Repository: incubator-systemml Updated Branches: refs/heads/master 474050653 -> 15b335edf
[SYSTEMML-1655] Improved codegen cost-based plan selection, minor fixes The patch makes the following two improvements to the codegen cost-based plan selector: (1) Cost model row aggregates: So far, the inputs to row aggregates with shared inputs under transpose operations, as they appear for example in t(X) %*% (X %*% v) have been double counted because the the cost model was not aware of the implicitly fused transpose operation. We now take this explicitly into account. (2) Overlapping outer product plans: Alternative outer product plans with equal number of plan references (but without multiple consumers) were not handled properly, leading to problems where wrong decisions on overlapping fusion plans destroyed one of multiple fusion patterns. We now take this into account and perform explicit pruning to handle such scenarios. Additionally, this patch also makes some minor fixes including: (1) proper main input selection for outer products that appear in the middle of a DAG, and (2) proper handling of row/column/row+column indexing in outer products with matrix side inputs. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/15b335ed Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/15b335ed Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/15b335ed Branch: refs/heads/master Commit: 15b335edf04ed8212650a6c5b2197b249bcc8452 Parents: 4740506 Author: Matthias Boehm <mboe...@gmail.com> Authored: Thu Jun 1 22:34:45 2017 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Thu Jun 1 22:34:45 2017 -0700 ---------------------------------------------------------------------- .../hops/codegen/template/CPlanMemoTable.java | 15 ++- .../template/PlanSelectionFuseCostBased.java | 32 +++++- .../codegen/template/TemplateOuterProduct.java | 42 ++++++-- .../test/integration/AutomatedTestBase.java | 7 ++ .../codegen/CompressedOuterProductTest.java | 104 +++++++++---------- .../codegen/CompressedRowAggregateTest.java | 4 +- .../codegen/CompressedOuterProductMain.dml | 1 - .../codegen/CompressedRowAggregateMain.dml | 1 - 8 files changed, 135 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/15b335ed/src/main/java/org/apache/sysml/hops/codegen/template/CPlanMemoTable.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/CPlanMemoTable.java b/src/main/java/org/apache/sysml/hops/codegen/template/CPlanMemoTable.java index 375f695..19d5a30 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/CPlanMemoTable.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/CPlanMemoTable.java @@ -67,6 +67,15 @@ public class CPlanMemoTable .filter(p -> p.type==type).findAny().isPresent(); } + public int countEntries(long hopID) { + return get(hopID).size(); + } + + public int countEntries(long hopID, TemplateType type) { + return (int) get(hopID).stream() + .filter(p -> p.type==type).count(); + } + public boolean containsTopLevel(long hopID) { return !_plansBlacklist.contains(hopID) && getBest(hopID) != null; @@ -104,7 +113,7 @@ public class CPlanMemoTable public void remove(Hop hop, HashSet<MemoTableEntry> blackList) { _plans.put(hop.getHopID(), _plans.get(hop.getHopID()).stream() - .filter(p -> !blackList.contains(p)).collect(Collectors.toList())); + .filter(p -> !blackList.contains(p)).collect(Collectors.toList())); } public void setDistinct(long hopID, List<MemoTableEntry> plans) { @@ -286,6 +295,10 @@ public class CPlanMemoTable + ((input2 >= 0) ? 1 : 0) + ((input3 >= 0) ? 1 : 0); } + public int getPlanRefIndex() { + return (input1>=0) ? 0 : (input2>=0) ? + 1 : (input3>=0) ? 2 : -1; + } public long input(int index) { return (index==0) ? input1 : (index==1) ? input2 : input3; } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/15b335ed/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java b/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java index 5769fcb..0e301e8 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java @@ -457,6 +457,24 @@ public class PlanSelectionFuseCostBased extends PlanSelection } } + //prune suboptimal outer product plans that are dominated by outer product plans w/ same number of + //references but better fusion properties (e.g., for the patterns Y=X*(U%*%t(V)) and sum(Y*(U2%*%t(V2))), + //we'd prune sum(X*(U%*%t(V))*Z), Z=U2%*%t(V2) because this would unnecessarily destroy a fusion pattern. + for( Long hopID : partition ) { + if( memo.countEntries(hopID, TemplateType.OuterProdTpl) == 2 ) { + List<MemoTableEntry> entries = memo.get(hopID, TemplateType.OuterProdTpl); + MemoTableEntry me1 = entries.get(0); + MemoTableEntry me2 = entries.get(1); + MemoTableEntry rmEntry = TemplateOuterProduct.dropAlternativePlan(memo, me1, me2); + if( rmEntry != null ) { + memo.remove(memo._hopRefs.get(hopID), new HashSet<MemoTableEntry>(Arrays.asList(rmEntry))); + memo._plansBlacklist.remove(rmEntry.input(rmEntry.getPlanRefIndex())); + if( LOG.isTraceEnabled() ) + LOG.trace("Removed dominated outer product memo table entry: " + rmEntry); + } + } + } + //if no materialization points, use basic fuse-all w/ partition awareness if( M == null || M.isEmpty() ) { for( Long hopID : R ) @@ -655,7 +673,7 @@ public class PlanSelectionFuseCostBased extends PlanSelection ArrayList<Long> M, boolean[] plan, HashMap<Long, Double> computeCosts) { //high level heuristic: every hop or fused operator has the following cost: - //WRITE + min(COMPUTE, READ), where WRITE costs are given by the output size, + //WRITE + max(COMPUTE, READ), where WRITE costs are given by the output size, //READ costs by the input sizes, and COMPUTE by operation specific FLOP //counts times number of cells of main input, disregarding sparsity for now. @@ -714,10 +732,12 @@ public class PlanSelectionFuseCostBased extends PlanSelection Hop c = current.getInput().get(i); if( best!=null && best.isPlanRef(i) ) costs += rGetPlanCosts(memo, c, visited, partition, M, plan, computeCosts, costVect, best.type); + else if( best!=null && isImplicitlyFused(current, i, best.type) ) + costVect.addInputSize(c.getInput().get(0).getHopID(), Math.max(c.getDim1(),1)*Math.max(c.getDim2(),1)); else { //include children and I/O costs costs += rGetPlanCosts(memo, c, visited, partition, M, plan, computeCosts, null, null); if( costVect != null && c.getDataType().isMatrix() ) - costVect.addInputSize( c.getHopID(), Math.max(c.getDim1(),1)*Math.max(c.getDim2(),1)); + costVect.addInputSize(c.getHopID(), Math.max(c.getDim1(),1)*Math.max(c.getDim2(),1)); } } @@ -725,7 +745,7 @@ public class PlanSelectionFuseCostBased extends PlanSelection if( partition.contains(current.getHopID()) ) { if( opened ) { if( LOG.isTraceEnabled() ) - LOG.trace("Cost vector for fused operator: "+costVect); + LOG.trace("Cost vector for fused operator (hop "+current.getHopID()+"): "+costVect); costs += costVect.outSize * 8 / WRITE_BANDWIDTH; //time for output write costs += Math.max( costVect.computeCosts*costVect.getMaxInputSize()/ COMPUTE_BANDWIDTH, @@ -900,6 +920,12 @@ public class PlanSelectionFuseCostBased extends PlanSelection return ret; } + private static boolean isImplicitlyFused(Hop hop, int index, TemplateType type) { + return type == TemplateType.RowTpl + && HopRewriteUtils.isMatrixMultiply(hop) && index==0 + && HopRewriteUtils.isTransposeOperation(hop.getInput().get(index)); + } + private static class CostVector { public final long ID; public final double outSize; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/15b335ed/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java index aa4fc5a..1dfe773 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java @@ -27,7 +27,6 @@ import java.util.LinkedList; import org.apache.sysml.hops.AggBinaryOp; import org.apache.sysml.hops.AggUnaryOp; import org.apache.sysml.hops.BinaryOp; -import org.apache.sysml.hops.DataOp; import org.apache.sysml.hops.Hop; import org.apache.sysml.hops.UnaryOp; import org.apache.sysml.hops.Hop.AggOp; @@ -166,16 +165,14 @@ public class TemplateOuterProduct extends TemplateBase { CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID()); String primitiveOpName = ((BinaryOp)hop).getOp().toString(); - if( (cdata1.getNumRows() > 1 && cdata1.getNumCols() == 1) || (cdata1.getNumRows() == 1 && cdata1.getNumCols() > 1) ) { - cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R); - } - if( (cdata2.getNumRows() > 1 && cdata2.getNumCols() == 1) || (cdata2.getNumRows() == 1 && cdata2.getNumCols() > 1) ) { - cdata2 = new CNodeUnary(cdata2, UnaryType.LOOKUP_R); + if( HopRewriteUtils.isEqualSize(hop.getInput().get(0), hop.getInput().get(1)) ) { + Hop main = hop.getInput().get((cdata1 instanceof CNodeData) ? 0 : 1); + inHops2.put("_X", main); } - if( HopRewriteUtils.isEqualSize(hop.getInput().get(0), hop.getInput().get(1)) - && hop.getInput().get(0) instanceof DataOp ) - inHops2.put("_X", hop.getInput().get(0)); + //add lookups if required + cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0)); + cdata2 = TemplateUtils.wrapLookupIfNecessary(cdata2, hop.getInput().get(1)); out = new CNodeBinary(cdata1, cdata2, BinType.valueOf(primitiveOpName)); } @@ -184,11 +181,11 @@ public class TemplateOuterProduct extends TemplateBase { CNode cdata1 = tmp.get(hop.getInput().get(0).getHopID()); CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID()); - //handle tanspose in outer or final product + //handle transpose in outer or final product cdata1 = TemplateUtils.skipTranspose(cdata1, hop.getInput().get(0), tmp, compileLiterals); cdata2 = TemplateUtils.skipTranspose(cdata2, hop.getInput().get(1), tmp, compileLiterals); - //outerproduct U%*%t(V), see open + //outer product U%*%t(V), see open if( HopRewriteUtils.isOuterProductLikeMM(hop) ) { //keep U and V for later reference @@ -220,4 +217,27 @@ public class TemplateOuterProduct extends TemplateBase { tmp.put(hop.getHopID(), out); } + + protected static MemoTableEntry dropAlternativePlan(CPlanMemoTable memo, MemoTableEntry me1, MemoTableEntry me2) { + //if there are two alternative sub plans with references to disjoint outer product plans + //drop the one that would render the other invalid + if( me1.countPlanRefs()==1 && me2.countPlanRefs()==1 + && me1.getPlanRefIndex() != me2.getPlanRefIndex() ) + { + Hop c1 = memo._hopRefs.get(me1.input(me1.getPlanRefIndex())); + Hop c2 = memo._hopRefs.get(me2.input(me2.getPlanRefIndex())); + + if( memo.contains(c1.getHopID(), TemplateType.OuterProdTpl) + && memo.contains(c2.getHopID(), TemplateType.OuterProdTpl) ) + { + if( HopRewriteUtils.isBinaryMatrixMatrixOperation(c1) + && HopRewriteUtils.isBinary(c1, OpOp2.MULT, OpOp2.DIV) ) + return me1; + if( HopRewriteUtils.isBinaryMatrixMatrixOperation(c2) + && HopRewriteUtils.isBinary(c2, OpOp2.MULT, OpOp2.DIV) ) + return me2; + } + } + return null; + } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/15b335ed/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java index ddd0f6d..0e56655 100644 --- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java @@ -1821,6 +1821,13 @@ public abstract class AutomatedTestBase return true; return false; } + + protected boolean heavyHittersContainsSubString(String str, int minCount) { + int count = 0; + for( String opcode : Statistics.getCPHeavyHitterOpCodes()) + count += opcode.contains(str) ? 1 : 0; + return (count >= minCount); + } /** * Create a SystemML-preferred Spark Session. http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/15b335ed/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java index 8c3a695..1f641f9 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java @@ -67,128 +67,128 @@ public class CompressedOuterProductTest extends AutomatedTestBase TestUtils.clearAssertionInformation(); addTestConfiguration( TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "R" }) ); } - + @Test - public void testCompressedCellwiseMainDenseConstCP() { - testCompressedCellwise( TEST_NAME1, SparsityType.DENSE, ValueType.CONST, ExecType.CP ); + public void testCompressedOuterProductMainDenseConstCP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.DENSE, ValueType.CONST, ExecType.CP ); } @Test - public void testCompressedCellwiseMainDenseRandCP() { - testCompressedCellwise( TEST_NAME1, SparsityType.DENSE, ValueType.RAND, ExecType.CP ); + public void testCompressedOuterProductMainDenseRandCP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.DENSE, ValueType.RAND, ExecType.CP ); } @Test - public void testCompressedCellwiseMainDenseRand2CP() { - testCompressedCellwise( TEST_NAME1, SparsityType.DENSE, ValueType.RAND_ROUND_DDC, ExecType.CP ); + public void testCompressedOuterProductMainDenseRand2CP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.DENSE, ValueType.RAND_ROUND_DDC, ExecType.CP ); } @Test - public void testCompressedCellwiseMainDenseRand3CP() { - testCompressedCellwise( TEST_NAME1, SparsityType.DENSE, ValueType.RAND_ROUND_OLE, ExecType.CP ); + public void testCompressedOuterProductMainDenseRand3CP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.DENSE, ValueType.RAND_ROUND_OLE, ExecType.CP ); } @Test - public void testCompressedCellwiseMainSparseConstCP() { - testCompressedCellwise( TEST_NAME1, SparsityType.SPARSE, ValueType.CONST, ExecType.CP ); + public void testCompressedOuterProductMainSparseConstCP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.SPARSE, ValueType.CONST, ExecType.CP ); } @Test - public void testCompressedCellwiseMainSparseRandCP() { - testCompressedCellwise( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND, ExecType.CP ); + public void testCompressedOuterProductMainSparseRandCP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND, ExecType.CP ); } @Test - public void testCompressedCellwiseMainSparseRand2CP() { - testCompressedCellwise( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND_ROUND_DDC, ExecType.CP ); + public void testCompressedOuterProductMainSparseRand2CP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND_ROUND_DDC, ExecType.CP ); } @Test - public void testCompressedCellwiseMainSparseRand3CP() { - testCompressedCellwise( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, ExecType.CP ); + public void testCompressedOuterProductMainSparseRand3CP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, ExecType.CP ); } @Test - public void testCompressedCellwiseMainEmptyConstCP() { - testCompressedCellwise( TEST_NAME1, SparsityType.EMPTY, ValueType.CONST, ExecType.CP ); + public void testCompressedOuterProductMainEmptyConstCP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.EMPTY, ValueType.CONST, ExecType.CP ); } @Test - public void testCompressedCellwiseMainEmptyRandCP() { - testCompressedCellwise( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND, ExecType.CP ); + public void testCompressedOuterProductMainEmptyRandCP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND, ExecType.CP ); } @Test - public void testCompressedCellwiseMainEmptyRand2CP() { - testCompressedCellwise( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND_ROUND_DDC, ExecType.CP ); + public void testCompressedOuterProductMainEmptyRand2CP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND_ROUND_DDC, ExecType.CP ); } @Test - public void testCompressedCellwiseMainEmptyRand3CP() { - testCompressedCellwise( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND_ROUND_OLE, ExecType.CP ); + public void testCompressedOuterProductMainEmptyRand3CP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND_ROUND_OLE, ExecType.CP ); } @Test - public void testCompressedCellwiseMainDenseConstSP() { - testCompressedCellwise( TEST_NAME1, SparsityType.DENSE, ValueType.CONST, ExecType.SPARK ); + public void testCompressedOuterProductMainDenseConstSP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.DENSE, ValueType.CONST, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainDenseRandSP() { - testCompressedCellwise( TEST_NAME1, SparsityType.DENSE, ValueType.RAND, ExecType.SPARK ); + public void testCompressedOuterProductMainDenseRandSP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.DENSE, ValueType.RAND, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainDenseRand2SP() { - testCompressedCellwise( TEST_NAME1, SparsityType.DENSE, ValueType.RAND_ROUND_DDC, ExecType.SPARK ); + public void testCompressedOuterProductMainDenseRand2SP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.DENSE, ValueType.RAND_ROUND_DDC, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainDenseRand3SP() { - testCompressedCellwise( TEST_NAME1, SparsityType.DENSE, ValueType.RAND_ROUND_OLE, ExecType.SPARK ); + public void testCompressedOuterProductMainDenseRand3SP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.DENSE, ValueType.RAND_ROUND_OLE, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainSparseConstSP() { - testCompressedCellwise( TEST_NAME1, SparsityType.SPARSE, ValueType.CONST, ExecType.SPARK ); + public void testCompressedOuterProductMainSparseConstSP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.SPARSE, ValueType.CONST, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainSparseRandSP() { - testCompressedCellwise( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND, ExecType.SPARK ); + public void testCompressedCompressedOuterProductMainSparseRandSP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainSparseRand2SP() { - testCompressedCellwise( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND_ROUND_DDC, ExecType.SPARK ); + public void testCompressedCompressedOuterProductMainSparseRand2SP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND_ROUND_DDC, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainSparseRand3SP() { - testCompressedCellwise( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, ExecType.SPARK ); + public void testCompressedCompressedOuterProductMainSparseRand3SP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.SPARSE, ValueType.RAND_ROUND_OLE, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainEmptyConstSP() { - testCompressedCellwise( TEST_NAME1, SparsityType.EMPTY, ValueType.CONST, ExecType.SPARK ); + public void testCompressedCompressedOuterProductMainEmptyConstSP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.EMPTY, ValueType.CONST, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainEmptyRandSP() { - testCompressedCellwise( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND, ExecType.SPARK ); + public void testCompressedCompressedOuterProductMainEmptyRandSP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainEmptyRand2SP() { - testCompressedCellwise( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND_ROUND_DDC, ExecType.SPARK ); + public void testCompressedCompressedOuterProductMainEmptyRand2SP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND_ROUND_DDC, ExecType.SPARK ); } @Test - public void testCompressedCellwiseMainEmptyRand3SP() { - testCompressedCellwise( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND_ROUND_OLE, ExecType.SPARK ); + public void testCompressedCompressedOuterProductMainEmptyRand3SP() { + testCompressedCompressedOuterProduct( TEST_NAME1, SparsityType.EMPTY, ValueType.RAND_ROUND_OLE, ExecType.SPARK ); } - private void testCompressedCellwise(String testname, SparsityType stype, ValueType vtype, ExecType et) + private void testCompressedCompressedOuterProduct(String testname, SparsityType stype, ValueType vtype, ExecType et) { boolean oldRewrites = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; RUNTIME_PLATFORM platformOld = rtplatform; @@ -241,8 +241,8 @@ public class CompressedOuterProductTest extends AutomatedTestBase HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R"); HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R"); TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R"); - Assert.assertTrue(heavyHittersContainsSubString("spoofOP") - || heavyHittersContainsSubString("sp_spoofOP")); + Assert.assertTrue(heavyHittersContainsSubString("spoofOP", 4) + || heavyHittersContainsSubString("sp_spoofOP", 4)); } finally { rtplatform = platformOld; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/15b335ed/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedRowAggregateTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedRowAggregateTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedRowAggregateTest.java index 02afa42..27f1b7c 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedRowAggregateTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedRowAggregateTest.java @@ -241,8 +241,8 @@ public class CompressedRowAggregateTest extends AutomatedTestBase HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R"); HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R"); TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R"); - Assert.assertTrue(heavyHittersContainsSubString("spoofRA") - || heavyHittersContainsSubString("sp_spoofRA")); + Assert.assertTrue(heavyHittersContainsSubString("spoofRA", 2) + || heavyHittersContainsSubString("sp_spoofRA", 2)); } finally { rtplatform = platformOld; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/15b335ed/src/test/scripts/functions/codegen/CompressedOuterProductMain.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/CompressedOuterProductMain.dml b/src/test/scripts/functions/codegen/CompressedOuterProductMain.dml index c7596e8..f0137c7 100644 --- a/src/test/scripts/functions/codegen/CompressedOuterProductMain.dml +++ b/src/test/scripts/functions/codegen/CompressedOuterProductMain.dml @@ -27,7 +27,6 @@ V = matrix(0.0001, ncol(X), 10); R1 = X * (7 + (U %*% t(V) + 3)); R2 = (X * (7 + (U %*% t(V) + 3))) %*% V; R3 = t(U) %*% (X * (7 + (U %*% t(V) + 3))); -if(1==1) {} R = as.matrix(sum(R1 * (7 + (R2 %*% R3 + 3))/1e6)); write(R, $2) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/15b335ed/src/test/scripts/functions/codegen/CompressedRowAggregateMain.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/CompressedRowAggregateMain.dml b/src/test/scripts/functions/codegen/CompressedRowAggregateMain.dml index 3c82181..9c881cb 100644 --- a/src/test/scripts/functions/codegen/CompressedRowAggregateMain.dml +++ b/src/test/scripts/functions/codegen/CompressedRowAggregateMain.dml @@ -24,7 +24,6 @@ v = seq(1, ncol(X)); w = seq(1, nrow(X)); R1 = t(X) %*% (X %*% v); -if(1==1) {} R2 = t(X) %*% (w * (X %*% v)); R = (R1 + R2) / 1e4;
