Repository: incubator-systemml Updated Branches: refs/heads/master 640df186a -> d33537a93
[SYSTEMML-1438] Extended code generator (celltmpl w/ sumsq aggregation) This patch extend the code generator w/ support for sum_squared aggregation in cell templates, which is important if ran in default optimization level w/ existing fused operators. Furthermore, this also includes new test cases and a cleanup of imports in generated classes. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/d33537a9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/d33537a9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/d33537a9 Branch: refs/heads/master Commit: d33537a9390a4d6e5b052786456aa82c0bb7a646 Parents: 640df18 Author: Matthias Boehm <mboe...@gmail.com> Authored: Mon Mar 27 00:07:47 2017 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Mon Mar 27 00:12:56 2017 -0700 ---------------------------------------------------------------------- .../sysml/hops/codegen/cplan/CNodeCell.java | 31 +++++++--- .../hops/codegen/cplan/CNodeOuterProduct.java | 9 +-- .../sysml/hops/codegen/cplan/CNodeRowAgg.java | 6 +- .../hops/codegen/template/TemplateCell.java | 14 +++-- .../hops/codegen/template/TemplateUtils.java | 8 ++- .../sysml/hops/rewrite/HopRewriteUtils.java | 4 ++ .../sysml/runtime/codegen/SpoofCellwise.java | 34 ++++++++--- .../functions/codegen/CellwiseTmplTest.java | 31 ++++++++-- .../codegen/SystemML-config-codegen6.xml | 61 ++++++++++++++++++++ .../scripts/functions/codegen/cellwisetmpl9.R | 32 ++++++++++ .../scripts/functions/codegen/cellwisetmpl9.dml | 28 +++++++++ 11 files changed, 219 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeCell.java b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeCell.java index 527da28..1bbbd67 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeCell.java +++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeCell.java @@ -22,6 +22,7 @@ package org.apache.sysml.hops.codegen.cplan; import java.util.ArrayList; import java.util.Arrays; +import org.apache.sysml.hops.Hop.AggOp; import org.apache.sysml.hops.codegen.SpoofFusedOp.SpoofOutputDimsType; import org.apache.sysml.runtime.codegen.SpoofCellwise.CellType; @@ -29,25 +30,24 @@ public class CNodeCell extends CNodeTpl { private static final String TEMPLATE = "package codegen;\n" - + "import java.util.Arrays;\n" - + "import java.io.Serializable;\n" - + "import java.util.ArrayList;\n" + "import org.apache.sysml.runtime.codegen.LibSpoofPrimitives;\n" + "import org.apache.sysml.runtime.codegen.SpoofCellwise;\n" + + "import org.apache.sysml.runtime.codegen.SpoofCellwise.AggOp;\n" + "import org.apache.sysml.runtime.codegen.SpoofCellwise.CellType;\n" + "import org.apache.commons.math3.util.FastMath;\n" + "\n" + "public final class %TMP% extends SpoofCellwise {\n" + " public %TMP%() {\n" - + " super(CellType.%TYPE%, %SPARSE_SAFE%);\n" + + " super(CellType.%TYPE%, %AGG_OP%, %SPARSE_SAFE%);\n" + " }\n" + " protected double genexec( double a, double[][] b, double[] scalars, int m, int n, int rowIndex, int colIndex) { \n" + "%BODY_dense%" + " return %OUT%;\n" - + " } \n" - + "}"; + + " }\n" + + "}\n"; private CellType _type = null; + private AggOp _aggOp = null; private boolean _sparseSafe = false; private boolean _requiresCastdtm = false; private boolean _multipleConsumers = false; @@ -73,6 +73,15 @@ public class CNodeCell extends CNodeTpl return _type; } + public void setAggOp(AggOp aggop) { + _aggOp = aggop; + _hash = 0; + } + + public AggOp getAggOp() { + return _aggOp; + } + public void setSparseSafe(boolean flag) { _sparseSafe = flag; } @@ -110,6 +119,7 @@ public class CNodeCell extends CNodeTpl //replace meta data information tmp = tmp.replaceAll("%TYPE%", getCellType().name()); + tmp = tmp.replaceAll("%AGG_OP%", (_aggOp!=null) ? "AggOp."+_aggOp.name() : "null" ); tmp = tmp.replaceAll("%SPARSE_SAFE%", String.valueOf(isSparseSafe())); return tmp; @@ -146,10 +156,11 @@ public class CNodeCell extends CNodeTpl if( _hash == 0 ) { int h1 = super.hashCode(); int h2 = _type.hashCode(); - int h3 = Boolean.valueOf(_sparseSafe).hashCode(); - int h4 = Boolean.valueOf(_requiresCastdtm).hashCode(); + int h3 = (_aggOp!=null) ? _aggOp.hashCode() : 0; + int h4 = Boolean.valueOf(_sparseSafe).hashCode(); + int h5 = Boolean.valueOf(_requiresCastdtm).hashCode(); //note: _multipleConsumers irrelevant for plan comparison - _hash = Arrays.hashCode(new int[]{h1,h2,h3,h4}); + _hash = Arrays.hashCode(new int[]{h1,h2,h3,h4,h5}); } return _hash; } @@ -162,6 +173,7 @@ public class CNodeCell extends CNodeTpl CNodeCell that = (CNodeCell)o; return super.equals(that) && _type == that._type + && _aggOp == that._aggOp && _sparseSafe == that._sparseSafe && _requiresCastdtm == that._requiresCastdtm && equalInputReferences( @@ -173,6 +185,7 @@ public class CNodeCell extends CNodeTpl StringBuilder sb = new StringBuilder(); sb.append("SPOOF CELLWISE [type="); sb.append(_type.name()); + sb.append(", aggOp="+((_aggOp!=null) ? _aggOp.name() : "null")); sb.append(", sparseSafe="+_sparseSafe); sb.append(", castdtm="+_requiresCastdtm); sb.append(", mc="+_multipleConsumers); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeOuterProduct.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeOuterProduct.java b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeOuterProduct.java index e9d9008..0dd6d62 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeOuterProduct.java +++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeOuterProduct.java @@ -30,8 +30,6 @@ public class CNodeOuterProduct extends CNodeTpl { private static final String TEMPLATE = "package codegen;\n" - + "import java.util.Arrays;\n" - + "import java.util.ArrayList;\n" + "import org.apache.sysml.runtime.codegen.LibSpoofPrimitives;\n" + "import org.apache.sysml.runtime.codegen.SpoofOuterProduct;\n" + "import org.apache.sysml.runtime.codegen.SpoofOuterProduct.OutProdType;\n" @@ -43,13 +41,12 @@ public class CNodeOuterProduct extends CNodeTpl + " }\n" + " protected void genexecDense( double a, double[] a1, int a1i, double[] a2, int a2i, double[][] b, double[] scalars, double[] c, int ci, int n, int m, int k, int rowIndex, int colIndex) { \n" + "%BODY_dense%" - + " } \n" + + " }\n" + " protected double genexecCellwise( double a, double[] a1, int a1i, double[] a2, int a2i, double[][] b, double[] scalars, int n, int m, int k, int rowIndex, int colIndex) { \n" + "%BODY_cellwise%" + " return %OUT_cellwise%;\n" - + " } \n" - - + "}"; + + " }\n" + + "}\n"; private OutProdType _type = null; private boolean _transposeOutput = false; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeRowAgg.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeRowAgg.java b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeRowAgg.java index 4f13c53..7073141 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeRowAgg.java +++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeRowAgg.java @@ -27,8 +27,6 @@ public class CNodeRowAgg extends CNodeTpl { private static final String TEMPLATE = "package codegen;\n" - + "import java.util.Arrays;\n" - + "import java.util.ArrayList;\n" + "import org.apache.sysml.runtime.codegen.LibSpoofPrimitives;\n" + "import org.apache.sysml.runtime.codegen.SpoofRowAggregate;\n" + "\n" @@ -38,10 +36,10 @@ public class CNodeRowAgg extends CNodeTpl + " }\n" + " protected void genexecRowDense( double[] a, int ai, double[][] b, double[] scalars, double[] c, int len, int rowIndex ) { \n" + "%BODY_dense%" - + " } \n" + + " }\n" + " protected void genexecRowSparse( double[] avals, int[] aix, int ai, double[][] b, double[] scalars, double[] c, int len, int rowIndex ) { \n" + "%BODY_sparse%" - + " } \n" + + " }\n" + "}\n"; public CNodeRowAgg(ArrayList<CNode> inputs, CNode output ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java index da74db2..f86bf69 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java @@ -31,7 +31,6 @@ import org.apache.sysml.hops.AggUnaryOp; import org.apache.sysml.hops.BinaryOp; import org.apache.sysml.hops.Hop; import org.apache.sysml.hops.UnaryOp; -import org.apache.sysml.hops.Hop.AggOp; import org.apache.sysml.hops.Hop.Direction; import org.apache.sysml.hops.Hop.OpOp2; import org.apache.sysml.hops.IndexingOp; @@ -67,7 +66,8 @@ public class TemplateCell extends TemplateBase @Override public boolean fuse(Hop hop, Hop input) { return !isClosed() && (isValidOperation(hop) - || (HopRewriteUtils.isSum(hop) && ((AggUnaryOp) hop).getDirection()!= Direction.Col) + || ((HopRewriteUtils.isSum(hop)||HopRewriteUtils.isSumSq(hop)) + && ((AggUnaryOp) hop).getDirection()!= Direction.Col) || (HopRewriteUtils.isMatrixMultiply(hop) && hop.getDim1()==1 && hop.getDim2()==1) && HopRewriteUtils.isTransposeOperation(hop.getInput().get(0))); } @@ -81,7 +81,8 @@ public class TemplateCell extends TemplateBase @Override public CloseType close(Hop hop) { //need to close cell tpl after aggregation, see fuse for exact properties - if( (HopRewriteUtils.isSum(hop) && ((AggUnaryOp) hop).getDirection()!= Direction.Col) + if( ((HopRewriteUtils.isSum(hop)||HopRewriteUtils.isSumSq(hop)) + && ((AggUnaryOp) hop).getDirection()!= Direction.Col) || (HopRewriteUtils.isMatrixMultiply(hop) && hop.getDim1()==1 && hop.getDim2()==1) ) return CloseType.CLOSED_VALID; else if( hop instanceof AggUnaryOp || hop instanceof AggBinaryOp ) @@ -114,6 +115,7 @@ public class TemplateCell extends TemplateBase CNode output = tmp.get(hop.getHopID()); CNodeCell tpl = new CNodeCell(inputs, output); tpl.setCellType(TemplateUtils.getCellType(hop)); + tpl.setAggOp(TemplateUtils.getAggOp(hop)); tpl.setSparseSafe((HopRewriteUtils.isBinary(hop, OpOp2.MULT) && hop.getInput().contains(sinHops.get(0))) || (HopRewriteUtils.isBinary(hop, OpOp2.DIV) && hop.getInput().get(0) == sinHops.get(0))); tpl.setRequiresCastDtm(hop instanceof AggBinaryOp); @@ -211,10 +213,10 @@ public class TemplateCell extends TemplateBase { out = tmp.get(hop.getInput().get(0).getHopID()); } - else if( hop instanceof AggUnaryOp && ((AggUnaryOp)hop).getOp() == AggOp.SUM - && (((AggUnaryOp) hop).getDirection() == Direction.RowCol - || ((AggUnaryOp) hop).getDirection() == Direction.Row) ) + else if( hop instanceof AggUnaryOp ) { + //aggregation handled in template implementation (note: we do not compile + //^2 of SUM_SQ into the operator to simplify the detection of single operators) out = tmp.get(hop.getInput().get(0).getHopID()); } else if( hop instanceof AggBinaryOp ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java index 4f9a5c7..dc42eb9 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java @@ -31,6 +31,7 @@ import org.apache.sysml.hops.BinaryOp; import org.apache.sysml.hops.Hop; import org.apache.sysml.hops.LiteralOp; import org.apache.sysml.hops.TernaryOp; +import org.apache.sysml.hops.Hop.AggOp; import org.apache.sysml.hops.Hop.Direction; import org.apache.sysml.hops.UnaryOp; import org.apache.sysml.hops.codegen.cplan.CNode; @@ -191,10 +192,15 @@ public class TemplateUtils public static CellType getCellType(Hop hop) { return (hop instanceof AggBinaryOp) ? CellType.FULL_AGG : - HopRewriteUtils.isSum(hop) ? ((((AggUnaryOp) hop).getDirection() == Direction.RowCol) ? + (hop instanceof AggUnaryOp) ? ((((AggUnaryOp) hop).getDirection() == Direction.RowCol) ? CellType.FULL_AGG : CellType.ROW_AGG) : CellType.NO_AGG; } + public static AggOp getAggOp(Hop hop) { + return (hop instanceof AggUnaryOp) ? ((AggUnaryOp)hop).getOp() : + (hop instanceof AggBinaryOp) ? AggOp.SUM : null; + } + public static OutProdType getOuterProductType(Hop X, Hop U, Hop V, Hop out) { if( out.getDataType() == DataType.SCALAR ) return OutProdType.AGG_OUTER_PRODUCT; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java index 8478fca..8ea0355 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java @@ -799,6 +799,10 @@ public class HopRewriteUtils return (hop instanceof AggUnaryOp && ((AggUnaryOp)hop).getOp()==AggOp.SUM); } + public static boolean isSumSq(Hop hop) { + return (hop instanceof AggUnaryOp && ((AggUnaryOp)hop).getOp()==AggOp.SUM_SQ); + } + public static boolean isNonZeroIndicator(Hop pred, Hop hop ) { if( pred instanceof BinaryOp && ((BinaryOp)pred).getOp()==OpOp2.NOTEQUAL http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java index f75aed6..f27564b 100644 --- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java +++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java @@ -29,7 +29,9 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.sysml.runtime.DMLRuntimeException; +import org.apache.sysml.runtime.functionobjects.KahanFunction; import org.apache.sysml.runtime.functionobjects.KahanPlus; +import org.apache.sysml.runtime.functionobjects.KahanPlusSq; import org.apache.sysml.runtime.instructions.cp.DoubleObject; import org.apache.sysml.runtime.instructions.cp.KahanObject; import org.apache.sysml.runtime.instructions.cp.ScalarObject; @@ -48,11 +50,19 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl ROW_AGG, } + //redefinition of Hop.AggOp for cleaner imports in generate class + public enum AggOp { + SUM, + SUM_SQ, + } + private final CellType _type; + private final AggOp _aggOp; private final boolean _sparseSafe; - public SpoofCellwise(CellType type, boolean sparseSafe) { + public SpoofCellwise(CellType type, AggOp aggOp, boolean sparseSafe) { _type = type; + _aggOp = aggOp; _sparseSafe = sparseSafe; } @@ -64,6 +74,16 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl return _sparseSafe; } + private KahanFunction getAggFunction() { + switch( _aggOp ) { + case SUM: return KahanPlus.getKahanPlusFnObject(); + case SUM_SQ: return KahanPlusSq.getKahanPlusSqFnObject(); + default: + throw new RuntimeException("Unsupported " + + "aggregation type: "+_aggOp.name()); + } + } + @Override public ScalarObject execute(ArrayList<MatrixBlock> inputs, ArrayList<ScalarObject> scalarObjects, int k) throws DMLRuntimeException @@ -202,11 +222,9 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl private double executeDenseAndAgg(double[] a, double[][] b, double[] scalars, int m, int n, boolean sparseSafe, int rl, int ru) { KahanObject kbuff = new KahanObject(0, 0); - KahanPlus kplus = KahanPlus.getKahanPlusFnObject(); + KahanFunction kplus = getAggFunction(); if( a == null && !sparseSafe ) { //empty - //note: we can't determine sparse-safeness by executing the operator once - //as the output might change with different row indices for( int i=rl; i<ru; i++ ) for( int j=0; j<n; j++ ) kplus.execute2(kbuff, genexec( 0, b, scalars, m, n, i, j )); @@ -248,11 +266,9 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl else if( _type == CellType.ROW_AGG ) { KahanObject kbuff = new KahanObject(0, 0); - KahanPlus kplus = KahanPlus.getKahanPlusFnObject(); + KahanFunction kplus = getAggFunction(); if( a == null && !sparseSafe ) { //empty - //note: we can't determine sparse-safeness by executing the operator once - //as the output might change with different row indices for( int i=rl; i<ru; i++ ) { kbuff.set(0, 0); for( int j=0; j<n; j++ ) @@ -279,7 +295,7 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl private double executeSparseAndAgg(SparseBlock sblock, double[][] b, double[] scalars, int m, int n, boolean sparseSafe, int rl, int ru) { KahanObject kbuff = new KahanObject(0, 0); - KahanPlus kplus = KahanPlus.getKahanPlusFnObject(); + KahanFunction kplus = getAggFunction(); if( sparseSafe ) { if( sblock != null ) { @@ -337,7 +353,7 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl else if( _type == CellType.ROW_AGG ) { KahanObject kbuff = new KahanObject(0, 0); - KahanPlus kplus = KahanPlus.getKahanPlusFnObject(); + KahanFunction kplus = getAggFunction(); if( sparseSafe ) { if( sblock != null ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java index 32105fd..3be9da8 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java @@ -44,18 +44,20 @@ public class CellwiseTmplTest extends AutomatedTestBase private static final String TEST_NAME6 = TEST_NAME+6; private static final String TEST_NAME7 = TEST_NAME+7; private static final String TEST_NAME8 = TEST_NAME+8; + private static final String TEST_NAME9 = TEST_NAME+9; //sum((X + 7 * Y)^2) private static final String TEST_DIR = "functions/codegen/"; private static final String TEST_CLASS_DIR = TEST_DIR + CellwiseTmplTest.class.getSimpleName() + "/"; - private final static String TEST_CONF = "SystemML-config-codegen.xml"; - private final static File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF); + private final static String TEST_CONF6 = "SystemML-config-codegen6.xml"; + private final static String TEST_CONF7 = "SystemML-config-codegen.xml"; + private static String TEST_CONF = TEST_CONF7; private static final double eps = Math.pow(10, -10); @Override public void setUp() { TestUtils.clearAssertionInformation(); - for( int i=1; i<=8; i++ ) { + for( int i=1; i<=9; i++ ) { addTestConfiguration( TEST_NAME+i, new TestConfiguration( TEST_CLASS_DIR, TEST_NAME+i, new String[] {String.valueOf(i)}) ); } @@ -101,6 +103,11 @@ public class CellwiseTmplTest extends AutomatedTestBase public void testCodegenCellwiseRewrite8() { testCodegenIntegration( TEST_NAME8, true, ExecType.CP ); } + + @Test + public void testCodegenCellwiseRewrite9() { + testCodegenIntegration( TEST_NAME9, true, ExecType.CP ); + } @Test public void testCodegenCellwise1() { @@ -142,6 +149,11 @@ public class CellwiseTmplTest extends AutomatedTestBase public void testCodegenCellwise8() { testCodegenIntegration( TEST_NAME8, false, ExecType.CP ); } + + @Test + public void testCodegenCellwise9() { + testCodegenIntegration( TEST_NAME9, false, ExecType.CP ); + } @Test public void testCodegenCellwiseRewrite1_sp() { @@ -158,10 +170,16 @@ public class CellwiseTmplTest extends AutomatedTestBase testCodegenIntegration( TEST_NAME8, true, ExecType.SPARK ); } + @Test + public void testCodegenCellwiseRewrite9_sp() { + testCodegenIntegration( TEST_NAME9, true, ExecType.SPARK ); + } + private void testCodegenIntegration( String testname, boolean rewrites, ExecType instType ) { boolean oldRewrites = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; + String oldTestConf = TEST_CONF; switch( instType ){ case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break; @@ -172,6 +190,9 @@ public class CellwiseTmplTest extends AutomatedTestBase default: rtplatform = RUNTIME_PLATFORM.HYBRID; break; } + if( testname.equals(TEST_NAME9) ) + TEST_CONF = TEST_CONF6; + try { TestConfiguration config = getTestConfiguration(testname); @@ -189,7 +210,7 @@ public class CellwiseTmplTest extends AutomatedTestBase runTest(true, false, null, -1); runRScript(true); - if(testname.equals(TEST_NAME6) || testname.equals(TEST_NAME7) ) { + if(testname.equals(TEST_NAME6) || testname.equals(TEST_NAME7) || testname.equals(TEST_NAME9) ) { //compare scalars HashMap<CellIndex, Double> dmlfile = readDMLScalarFromHDFS("S"); HashMap<CellIndex, Double> rfile = readRScalarFromFS("S"); @@ -212,6 +233,7 @@ public class CellwiseTmplTest extends AutomatedTestBase OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldRewrites; OptimizerUtils.ALLOW_AUTO_VECTORIZATION = true; OptimizerUtils.ALLOW_OPERATOR_FUSION = true; + TEST_CONF = oldTestConf; } } @@ -222,6 +244,7 @@ public class CellwiseTmplTest extends AutomatedTestBase @Override protected File getConfigTemplateFile() { // Instrumentation in this test's output log to show custom configuration file used for template. + File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF); System.out.println("This test case overrides default configuration with " + TEST_CONF_FILE.getPath()); return TEST_CONF_FILE; } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml b/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml new file mode 100644 index 0000000..aa7f5bd --- /dev/null +++ b/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml @@ -0,0 +1,61 @@ +<!-- + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. +--> + +<root> + <!-- local fs tmp working directory--> + <localtmpdir>/tmp/systemml</localtmpdir> + + <!-- hdfs tmp working directory--> + <scratch>scratch_space</scratch> + + <!-- compiler optimization level, valid values: 0 | 1 | 2 | 3 | 4, default: 2 --> + <optlevel>6</optlevel> + + <!-- default number of reduce tasks per MR job, default: 2 x number of nodes --> + <numreducers>10</numreducers> + + <!-- override jvm reuse flag for specific MR jobs, valid values: true | false --> + <jvmreuse>false</jvmreuse> + + <!-- default block dim for binary block files --> + <defaultblocksize>1000</defaultblocksize> + + <!-- run systemml control program as yarn appmaster, in case of MR1 always falls back to client, please disable for debug mode --> + <dml.yarn.appmaster>false</dml.yarn.appmaster> + + <!-- maximum jvm heap size of the dml yarn appmaster in MB, the requested memory is 1.5x this parameter --> + <dml.yarn.appmaster.mem>2048</dml.yarn.appmaster.mem> + + <!-- maximum jvm heap size of the map/reduce tasks in MB, the requested memory is 1.5x this parameter, negative values ignored --> + <dml.yarn.mapreduce.mem>2048</dml.yarn.mapreduce.mem> + + <!-- yarn application submission queue, relevant for default capacity scheduler --> + <dml.yarn.app.queue>default</dml.yarn.app.queue> + + <!-- enables multi-threaded matrix multiplications in singlenode control program --> + <cp.parallel.matrixmult>true</cp.parallel.matrixmult> + + <!-- enables multi-threaded read/write of text formats in singlenode control program --> + <cp.parallel.textio>true</cp.parallel.textio> + + <!-- enables automatic code generation --> + <codegen.enabled>true</codegen.enabled> + <codegen.plancache>true</codegen.plancache> + <codegen.literals>1</codegen.literals> +</root> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/test/scripts/functions/codegen/cellwisetmpl9.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/cellwisetmpl9.R b/src/test/scripts/functions/codegen/cellwisetmpl9.R new file mode 100644 index 0000000..5d25389 --- /dev/null +++ b/src/test/scripts/functions/codegen/cellwisetmpl9.R @@ -0,0 +1,32 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args<-commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X = matrix(seq(7, 1006), 500, 2); +Y = matrix(seq(6, 1005), 500, 2); + +Z = X + 7 * Y; +R = sum(Z^2) + +write(R, paste(args[2],"S",sep="")) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/test/scripts/functions/codegen/cellwisetmpl9.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/cellwisetmpl9.dml b/src/test/scripts/functions/codegen/cellwisetmpl9.dml new file mode 100644 index 0000000..b86a086 --- /dev/null +++ b/src/test/scripts/functions/codegen/cellwisetmpl9.dml @@ -0,0 +1,28 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = matrix(seq(7, 1006), 500, 2); +Y = matrix(seq(6, 1005), 500, 2); + +Z = X + 7 * Y; +R = sum(Z^2) + +write(R, $1)
