Repository: incubator-systemml
Updated Branches:
  refs/heads/master 640df186a -> d33537a93


[SYSTEMML-1438] Extended code generator (celltmpl w/ sumsq aggregation)

This patch extend the code generator w/ support for sum_squared
aggregation in cell templates, which is important if ran in default
optimization level w/ existing fused operators. Furthermore, this also
includes new test cases and a cleanup of imports in generated classes.


Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/d33537a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/d33537a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/d33537a9

Branch: refs/heads/master
Commit: d33537a9390a4d6e5b052786456aa82c0bb7a646
Parents: 640df18
Author: Matthias Boehm <mboe...@gmail.com>
Authored: Mon Mar 27 00:07:47 2017 -0700
Committer: Matthias Boehm <mboe...@gmail.com>
Committed: Mon Mar 27 00:12:56 2017 -0700

----------------------------------------------------------------------
 .../sysml/hops/codegen/cplan/CNodeCell.java     | 31 +++++++---
 .../hops/codegen/cplan/CNodeOuterProduct.java   |  9 +--
 .../sysml/hops/codegen/cplan/CNodeRowAgg.java   |  6 +-
 .../hops/codegen/template/TemplateCell.java     | 14 +++--
 .../hops/codegen/template/TemplateUtils.java    |  8 ++-
 .../sysml/hops/rewrite/HopRewriteUtils.java     |  4 ++
 .../sysml/runtime/codegen/SpoofCellwise.java    | 34 ++++++++---
 .../functions/codegen/CellwiseTmplTest.java     | 31 ++++++++--
 .../codegen/SystemML-config-codegen6.xml        | 61 ++++++++++++++++++++
 .../scripts/functions/codegen/cellwisetmpl9.R   | 32 ++++++++++
 .../scripts/functions/codegen/cellwisetmpl9.dml | 28 +++++++++
 11 files changed, 219 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeCell.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeCell.java 
b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeCell.java
index 527da28..1bbbd67 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeCell.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeCell.java
@@ -22,6 +22,7 @@ package org.apache.sysml.hops.codegen.cplan;
 import java.util.ArrayList;
 import java.util.Arrays;
 
+import org.apache.sysml.hops.Hop.AggOp;
 import org.apache.sysml.hops.codegen.SpoofFusedOp.SpoofOutputDimsType;
 import org.apache.sysml.runtime.codegen.SpoofCellwise.CellType;
 
@@ -29,25 +30,24 @@ public class CNodeCell extends CNodeTpl
 {      
        private static final String TEMPLATE = 
                          "package codegen;\n"
-                       + "import java.util.Arrays;\n"
-                       + "import java.io.Serializable;\n"
-                       + "import java.util.ArrayList;\n"
                        + "import 
org.apache.sysml.runtime.codegen.LibSpoofPrimitives;\n"
                        + "import 
org.apache.sysml.runtime.codegen.SpoofCellwise;\n"
+                       + "import 
org.apache.sysml.runtime.codegen.SpoofCellwise.AggOp;\n"
                        + "import 
org.apache.sysml.runtime.codegen.SpoofCellwise.CellType;\n"
                        + "import org.apache.commons.math3.util.FastMath;\n"
                        + "\n"
                        + "public final class %TMP% extends SpoofCellwise {\n" 
                        + "  public %TMP%() {\n"
-                       + "    super(CellType.%TYPE%, %SPARSE_SAFE%);\n"
+                       + "    super(CellType.%TYPE%, %AGG_OP%, 
%SPARSE_SAFE%);\n"
                        + "  }\n"
                        + "  protected double genexec( double a, double[][] b, 
double[] scalars, int m, int n, int rowIndex, int colIndex) { \n"
                        + "%BODY_dense%"
                        + "    return %OUT%;\n"
-                       + "  } \n"
-                       + "}";
+                       + "  }\n"
+                       + "}\n";
        
        private CellType _type = null;
+       private AggOp _aggOp = null;
        private boolean _sparseSafe = false;
        private boolean _requiresCastdtm = false;
        private boolean _multipleConsumers = false;
@@ -73,6 +73,15 @@ public class CNodeCell extends CNodeTpl
                return _type;
        }
        
+       public void setAggOp(AggOp aggop) {
+               _aggOp = aggop;
+               _hash = 0;
+       }
+       
+       public AggOp getAggOp() {
+               return _aggOp;
+       }
+       
        public void setSparseSafe(boolean flag) {
                _sparseSafe = flag;
        }
@@ -110,6 +119,7 @@ public class CNodeCell extends CNodeTpl
                
                //replace meta data information
                tmp = tmp.replaceAll("%TYPE%", getCellType().name());
+               tmp = tmp.replaceAll("%AGG_OP%", (_aggOp!=null) ? 
"AggOp."+_aggOp.name() : "null" );
                tmp = tmp.replaceAll("%SPARSE_SAFE%", 
String.valueOf(isSparseSafe()));
                
                return tmp;
@@ -146,10 +156,11 @@ public class CNodeCell extends CNodeTpl
                if( _hash == 0 ) {
                        int h1 = super.hashCode();
                        int h2 = _type.hashCode();
-                       int h3 = Boolean.valueOf(_sparseSafe).hashCode();
-                       int h4 = Boolean.valueOf(_requiresCastdtm).hashCode();
+                       int h3 = (_aggOp!=null) ? _aggOp.hashCode() : 0;
+                       int h4 = Boolean.valueOf(_sparseSafe).hashCode();
+                       int h5 = Boolean.valueOf(_requiresCastdtm).hashCode();
                        //note: _multipleConsumers irrelevant for plan 
comparison
-                       _hash = Arrays.hashCode(new int[]{h1,h2,h3,h4});
+                       _hash = Arrays.hashCode(new int[]{h1,h2,h3,h4,h5});
                }
                return _hash;
        }
@@ -162,6 +173,7 @@ public class CNodeCell extends CNodeTpl
                CNodeCell that = (CNodeCell)o;
                return super.equals(that) 
                        && _type == that._type
+                       && _aggOp == that._aggOp
                        && _sparseSafe == that._sparseSafe
                        && _requiresCastdtm == that._requiresCastdtm
                        && equalInputReferences(
@@ -173,6 +185,7 @@ public class CNodeCell extends CNodeTpl
                StringBuilder sb = new StringBuilder();
                sb.append("SPOOF CELLWISE [type=");
                sb.append(_type.name());
+               sb.append(", aggOp="+((_aggOp!=null) ? _aggOp.name() : "null"));
                sb.append(", sparseSafe="+_sparseSafe);
                sb.append(", castdtm="+_requiresCastdtm);
                sb.append(", mc="+_multipleConsumers);

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeOuterProduct.java
----------------------------------------------------------------------
diff --git 
a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeOuterProduct.java 
b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeOuterProduct.java
index e9d9008..0dd6d62 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeOuterProduct.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeOuterProduct.java
@@ -30,8 +30,6 @@ public class CNodeOuterProduct extends CNodeTpl
 {      
        private static final String TEMPLATE = 
                          "package codegen;\n"
-                       + "import java.util.Arrays;\n"
-                       + "import java.util.ArrayList;\n"
                        + "import 
org.apache.sysml.runtime.codegen.LibSpoofPrimitives;\n"
                        + "import 
org.apache.sysml.runtime.codegen.SpoofOuterProduct;\n"
                        + "import 
org.apache.sysml.runtime.codegen.SpoofOuterProduct.OutProdType;\n"
@@ -43,13 +41,12 @@ public class CNodeOuterProduct extends CNodeTpl
                        + "  }\n"
                        + "  protected void genexecDense( double a, double[] 
a1, int a1i, double[] a2, int a2i, double[][] b, double[] scalars, double[] c, 
int ci, int n, int m, int k, int rowIndex, int colIndex) { \n"
                        + "%BODY_dense%"
-                       + "  } \n"
+                       + "  }\n"
                        + "  protected double genexecCellwise( double a, 
double[] a1, int a1i, double[] a2, int a2i, double[][] b, double[] scalars, int 
n, int m, int k, int rowIndex, int colIndex) { \n"
                        + "%BODY_cellwise%"
                        + "    return %OUT_cellwise%;\n"
-                       + "  } \n"
-                       
-                       + "}";
+                       + "  }\n"                       
+                       + "}\n";
        
        private OutProdType _type = null;
        private boolean _transposeOutput = false;

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeRowAgg.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeRowAgg.java 
b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeRowAgg.java
index 4f13c53..7073141 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeRowAgg.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/cplan/CNodeRowAgg.java
@@ -27,8 +27,6 @@ public class CNodeRowAgg extends CNodeTpl
 {
        private static final String TEMPLATE = 
                          "package codegen;\n"
-                       + "import java.util.Arrays;\n"
-                       + "import java.util.ArrayList;\n"
                        + "import 
org.apache.sysml.runtime.codegen.LibSpoofPrimitives;\n"
                        + "import 
org.apache.sysml.runtime.codegen.SpoofRowAggregate;\n"
                        + "\n"
@@ -38,10 +36,10 @@ public class CNodeRowAgg extends CNodeTpl
                        + "  }\n"
                        + "  protected void genexecRowDense( double[] a, int 
ai, double[][] b, double[] scalars, double[] c, int len, int rowIndex ) { \n"
                        + "%BODY_dense%"
-                       + "  } \n"
+                       + "  }\n"
                        + "  protected void genexecRowSparse( double[] avals, 
int[] aix, int ai, double[][] b, double[] scalars, double[] c, int len, int 
rowIndex ) { \n"
                        + "%BODY_sparse%"
-                       + "  } \n"                      
+                       + "  }\n"                       
                        + "}\n";
 
        public CNodeRowAgg(ArrayList<CNode> inputs, CNode output ) {

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
----------------------------------------------------------------------
diff --git 
a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java 
b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
index da74db2..f86bf69 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java
@@ -31,7 +31,6 @@ import org.apache.sysml.hops.AggUnaryOp;
 import org.apache.sysml.hops.BinaryOp;
 import org.apache.sysml.hops.Hop;
 import org.apache.sysml.hops.UnaryOp;
-import org.apache.sysml.hops.Hop.AggOp;
 import org.apache.sysml.hops.Hop.Direction;
 import org.apache.sysml.hops.Hop.OpOp2;
 import org.apache.sysml.hops.IndexingOp;
@@ -67,7 +66,8 @@ public class TemplateCell extends TemplateBase
        @Override
        public boolean fuse(Hop hop, Hop input) {
                return !isClosed() && (isValidOperation(hop) 
-                       || (HopRewriteUtils.isSum(hop) && ((AggUnaryOp) 
hop).getDirection()!= Direction.Col)
+                       || 
((HopRewriteUtils.isSum(hop)||HopRewriteUtils.isSumSq(hop)) 
+                               && ((AggUnaryOp) hop).getDirection()!= 
Direction.Col)
                        || (HopRewriteUtils.isMatrixMultiply(hop) && 
hop.getDim1()==1 && hop.getDim2()==1)
                                && 
HopRewriteUtils.isTransposeOperation(hop.getInput().get(0)));
        }
@@ -81,7 +81,8 @@ public class TemplateCell extends TemplateBase
        @Override
        public CloseType close(Hop hop) {
                //need to close cell tpl after aggregation, see fuse for exact 
properties
-               if( (HopRewriteUtils.isSum(hop) && ((AggUnaryOp) 
hop).getDirection()!= Direction.Col)
+               if( ((HopRewriteUtils.isSum(hop)||HopRewriteUtils.isSumSq(hop)) 
+                               && ((AggUnaryOp) hop).getDirection()!= 
Direction.Col)
                        || (HopRewriteUtils.isMatrixMultiply(hop) && 
hop.getDim1()==1 && hop.getDim2()==1) )
                        return CloseType.CLOSED_VALID;
                else if( hop instanceof AggUnaryOp || hop instanceof 
AggBinaryOp )
@@ -114,6 +115,7 @@ public class TemplateCell extends TemplateBase
                CNode output = tmp.get(hop.getHopID());
                CNodeCell tpl = new CNodeCell(inputs, output);
                tpl.setCellType(TemplateUtils.getCellType(hop));
+               tpl.setAggOp(TemplateUtils.getAggOp(hop));
                tpl.setSparseSafe((HopRewriteUtils.isBinary(hop, OpOp2.MULT) && 
hop.getInput().contains(sinHops.get(0)))
                                || (HopRewriteUtils.isBinary(hop, OpOp2.DIV) && 
hop.getInput().get(0) == sinHops.get(0)));
                tpl.setRequiresCastDtm(hop instanceof AggBinaryOp);
@@ -211,10 +213,10 @@ public class TemplateCell extends TemplateBase
                {
                        out = tmp.get(hop.getInput().get(0).getHopID());        
                }
-               else if( hop instanceof AggUnaryOp && ((AggUnaryOp)hop).getOp() 
== AggOp.SUM
-                       && (((AggUnaryOp) hop).getDirection() == 
Direction.RowCol 
-                       || ((AggUnaryOp) hop).getDirection() == Direction.Row) )
+               else if( hop instanceof AggUnaryOp )
                {
+                       //aggregation handled in template implementation (note: 
we do not compile 
+                       //^2 of SUM_SQ into the operator to simplify the 
detection of single operators)
                        out = tmp.get(hop.getInput().get(0).getHopID());
                }
                else if( hop instanceof AggBinaryOp ) {

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java
----------------------------------------------------------------------
diff --git 
a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java 
b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java
index 4f9a5c7..dc42eb9 100644
--- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java
+++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateUtils.java
@@ -31,6 +31,7 @@ import org.apache.sysml.hops.BinaryOp;
 import org.apache.sysml.hops.Hop;
 import org.apache.sysml.hops.LiteralOp;
 import org.apache.sysml.hops.TernaryOp;
+import org.apache.sysml.hops.Hop.AggOp;
 import org.apache.sysml.hops.Hop.Direction;
 import org.apache.sysml.hops.UnaryOp;
 import org.apache.sysml.hops.codegen.cplan.CNode;
@@ -191,10 +192,15 @@ public class TemplateUtils
        
        public static CellType getCellType(Hop hop) {
                return (hop instanceof AggBinaryOp) ? CellType.FULL_AGG :
-                       HopRewriteUtils.isSum(hop) ? ((((AggUnaryOp) 
hop).getDirection() == Direction.RowCol) ? 
+                       (hop instanceof AggUnaryOp) ? ((((AggUnaryOp) 
hop).getDirection() == Direction.RowCol) ? 
                        CellType.FULL_AGG : CellType.ROW_AGG) : CellType.NO_AGG;
        }
        
+       public static AggOp getAggOp(Hop hop) {
+               return (hop instanceof AggUnaryOp) ? ((AggUnaryOp)hop).getOp() :
+                       (hop instanceof AggBinaryOp) ? AggOp.SUM : null;
+       }
+       
        public static OutProdType getOuterProductType(Hop X, Hop U, Hop V, Hop 
out) {
                if( out.getDataType() == DataType.SCALAR ) 
                        return OutProdType.AGG_OUTER_PRODUCT;

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java 
b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
index 8478fca..8ea0355 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
@@ -799,6 +799,10 @@ public class HopRewriteUtils
                return (hop instanceof AggUnaryOp && 
((AggUnaryOp)hop).getOp()==AggOp.SUM);
        }
        
+       public static boolean isSumSq(Hop hop) {
+               return (hop instanceof AggUnaryOp && 
((AggUnaryOp)hop).getOp()==AggOp.SUM_SQ);
+       }
+       
        public static boolean isNonZeroIndicator(Hop pred, Hop hop )
        {
                if( pred instanceof BinaryOp && 
((BinaryOp)pred).getOp()==OpOp2.NOTEQUAL

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java 
b/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
index f75aed6..f27564b 100644
--- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
+++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java
@@ -29,7 +29,9 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
 import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.functionobjects.KahanFunction;
 import org.apache.sysml.runtime.functionobjects.KahanPlus;
+import org.apache.sysml.runtime.functionobjects.KahanPlusSq;
 import org.apache.sysml.runtime.instructions.cp.DoubleObject;
 import org.apache.sysml.runtime.instructions.cp.KahanObject;
 import org.apache.sysml.runtime.instructions.cp.ScalarObject;
@@ -48,11 +50,19 @@ public abstract class SpoofCellwise extends SpoofOperator 
implements Serializabl
                ROW_AGG,
        }
        
+       //redefinition of Hop.AggOp for cleaner imports in generate class
+       public enum AggOp {
+               SUM, 
+               SUM_SQ,
+       }
+       
        private final CellType _type;
+       private final AggOp _aggOp;
        private final boolean _sparseSafe;
        
-       public SpoofCellwise(CellType type, boolean sparseSafe) {
+       public SpoofCellwise(CellType type, AggOp aggOp, boolean sparseSafe) {
                _type = type;
+               _aggOp = aggOp;
                _sparseSafe = sparseSafe;
        }
        
@@ -64,6 +74,16 @@ public abstract class SpoofCellwise extends SpoofOperator 
implements Serializabl
                return _sparseSafe;
        }
        
+       private KahanFunction getAggFunction() {
+               switch( _aggOp ) {
+                       case SUM: return KahanPlus.getKahanPlusFnObject();
+                       case SUM_SQ: return 
KahanPlusSq.getKahanPlusSqFnObject();
+                       default:
+                               throw new RuntimeException("Unsupported "
+                                               + "aggregation type: 
"+_aggOp.name());
+               }
+       }
+               
        @Override
        public ScalarObject execute(ArrayList<MatrixBlock> inputs, 
ArrayList<ScalarObject> scalarObjects, int k) 
                throws DMLRuntimeException 
@@ -202,11 +222,9 @@ public abstract class SpoofCellwise extends SpoofOperator 
implements Serializabl
        private double executeDenseAndAgg(double[] a, double[][] b, double[] 
scalars, int m, int n, boolean sparseSafe, int rl, int ru) 
        {
                KahanObject kbuff = new KahanObject(0, 0);
-               KahanPlus kplus = KahanPlus.getKahanPlusFnObject();
+               KahanFunction kplus = getAggFunction();
 
                if( a == null && !sparseSafe ) { //empty
-                       //note: we can't determine sparse-safeness by executing 
the operator once 
-                       //as the output might change with different row indices
                        for( int i=rl; i<ru; i++ ) 
                                for( int j=0; j<n; j++ )
                                        kplus.execute2(kbuff, genexec( 0, b, 
scalars, m, n, i, j ));
@@ -248,11 +266,9 @@ public abstract class SpoofCellwise extends SpoofOperator 
implements Serializabl
                else if( _type == CellType.ROW_AGG )
                {
                        KahanObject kbuff = new KahanObject(0, 0);
-                       KahanPlus kplus = KahanPlus.getKahanPlusFnObject();
+                       KahanFunction kplus = getAggFunction();
 
                        if( a == null && !sparseSafe ) { //empty
-                               //note: we can't determine sparse-safeness by 
executing the operator once 
-                               //as the output might change with different row 
indices
                                for( int i=rl; i<ru; i++ ) { 
                                        kbuff.set(0, 0);
                                        for( int j=0; j<n; j++ )
@@ -279,7 +295,7 @@ public abstract class SpoofCellwise extends SpoofOperator 
implements Serializabl
        private double executeSparseAndAgg(SparseBlock sblock, double[][] b, 
double[] scalars, int m, int n, boolean sparseSafe, int rl, int ru) 
        {
                KahanObject kbuff = new KahanObject(0, 0);
-               KahanPlus kplus = KahanPlus.getKahanPlusFnObject();
+               KahanFunction kplus = getAggFunction();
                
                if( sparseSafe ) {
                        if( sblock != null ) {
@@ -337,7 +353,7 @@ public abstract class SpoofCellwise extends SpoofOperator 
implements Serializabl
                else if( _type == CellType.ROW_AGG ) 
                {
                        KahanObject kbuff = new KahanObject(0, 0);
-                       KahanPlus kplus = KahanPlus.getKahanPlusFnObject();
+                       KahanFunction kplus = getAggFunction();
 
                        if( sparseSafe ) {
                                if( sblock != null ) {

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
----------------------------------------------------------------------
diff --git 
a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
 
b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
index 32105fd..3be9da8 100644
--- 
a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
+++ 
b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java
@@ -44,18 +44,20 @@ public class CellwiseTmplTest extends AutomatedTestBase
        private static final String TEST_NAME6 = TEST_NAME+6;
        private static final String TEST_NAME7 = TEST_NAME+7;
        private static final String TEST_NAME8 = TEST_NAME+8;
+       private static final String TEST_NAME9 = TEST_NAME+9; //sum((X + 7 * 
Y)^2)
 
        private static final String TEST_DIR = "functions/codegen/";
        private static final String TEST_CLASS_DIR = TEST_DIR + 
CellwiseTmplTest.class.getSimpleName() + "/";
-       private final static String TEST_CONF = "SystemML-config-codegen.xml";
-       private final static File   TEST_CONF_FILE = new File(SCRIPT_DIR + 
TEST_DIR, TEST_CONF);
+       private final static String TEST_CONF6 = "SystemML-config-codegen6.xml";
+       private final static String TEST_CONF7 = "SystemML-config-codegen.xml";
+       private static String TEST_CONF = TEST_CONF7;
        
        private static final double eps = Math.pow(10, -10);
        
        @Override
        public void setUp() {
                TestUtils.clearAssertionInformation();
-               for( int i=1; i<=8; i++ ) {
+               for( int i=1; i<=9; i++ ) {
                        addTestConfiguration( TEST_NAME+i, new 
TestConfiguration(
                                        TEST_CLASS_DIR, TEST_NAME+i, new 
String[] {String.valueOf(i)}) );
                }
@@ -101,6 +103,11 @@ public class CellwiseTmplTest extends AutomatedTestBase
        public void testCodegenCellwiseRewrite8() {
                testCodegenIntegration( TEST_NAME8, true, ExecType.CP  );
        }
+       
+       @Test
+       public void testCodegenCellwiseRewrite9() {
+               testCodegenIntegration( TEST_NAME9, true, ExecType.CP  );
+       }
 
        @Test
        public void testCodegenCellwise1() {
@@ -142,6 +149,11 @@ public class CellwiseTmplTest extends AutomatedTestBase
        public void testCodegenCellwise8() {
                testCodegenIntegration( TEST_NAME8, false, ExecType.CP  );
        }
+       
+       @Test
+       public void testCodegenCellwise9() {
+               testCodegenIntegration( TEST_NAME9, false, ExecType.CP  );
+       }
 
        @Test
        public void testCodegenCellwiseRewrite1_sp() {
@@ -158,10 +170,16 @@ public class CellwiseTmplTest extends AutomatedTestBase
                testCodegenIntegration( TEST_NAME8, true, ExecType.SPARK );
        }
        
+       @Test
+       public void testCodegenCellwiseRewrite9_sp() {
+               testCodegenIntegration( TEST_NAME9, true, ExecType.SPARK );
+       }
+       
        private void testCodegenIntegration( String testname, boolean rewrites, 
ExecType instType )
        {       
                
                boolean oldRewrites = 
OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION;
+               String oldTestConf = TEST_CONF;
                
                switch( instType ){
                        case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break;
@@ -172,6 +190,9 @@ public class CellwiseTmplTest extends AutomatedTestBase
                        default: rtplatform = RUNTIME_PLATFORM.HYBRID; break;
                }
                
+               if( testname.equals(TEST_NAME9) )
+                       TEST_CONF = TEST_CONF6;
+               
                try
                {
                        TestConfiguration config = 
getTestConfiguration(testname);
@@ -189,7 +210,7 @@ public class CellwiseTmplTest extends AutomatedTestBase
                        runTest(true, false, null, -1); 
                        runRScript(true); 
                        
-                       if(testname.equals(TEST_NAME6) || 
testname.equals(TEST_NAME7) ) {
+                       if(testname.equals(TEST_NAME6) || 
testname.equals(TEST_NAME7) || testname.equals(TEST_NAME9) ) {
                                //compare scalars 
                                HashMap<CellIndex, Double> dmlfile = 
readDMLScalarFromHDFS("S");
                                HashMap<CellIndex, Double> rfile  = 
readRScalarFromFS("S");
@@ -212,6 +233,7 @@ public class CellwiseTmplTest extends AutomatedTestBase
                        OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = 
oldRewrites;
                        OptimizerUtils.ALLOW_AUTO_VECTORIZATION = true;
                        OptimizerUtils.ALLOW_OPERATOR_FUSION = true;
+                       TEST_CONF = oldTestConf;
                }
        }       
 
@@ -222,6 +244,7 @@ public class CellwiseTmplTest extends AutomatedTestBase
        @Override
        protected File getConfigTemplateFile() {
                // Instrumentation in this test's output log to show custom 
configuration file used for template.
+               File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, 
TEST_CONF);
                System.out.println("This test case overrides default 
configuration with " + TEST_CONF_FILE.getPath());
                return TEST_CONF_FILE;
        }

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml 
b/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml
new file mode 100644
index 0000000..aa7f5bd
--- /dev/null
+++ b/src/test/scripts/functions/codegen/SystemML-config-codegen6.xml
@@ -0,0 +1,61 @@
+<!--
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+-->
+
+<root>
+   <!-- local fs tmp working directory-->
+   <localtmpdir>/tmp/systemml</localtmpdir>
+
+   <!-- hdfs tmp working directory--> 
+   <scratch>scratch_space</scratch> 
+
+   <!-- compiler optimization level, valid values: 0 | 1 | 2 | 3 | 4, default: 
2 -->
+   <optlevel>6</optlevel>  
+
+   <!-- default number of reduce tasks per MR job, default: 2 x number of 
nodes -->
+   <numreducers>10</numreducers> 
+   
+   <!-- override jvm reuse flag for specific MR jobs, valid values: true | 
false  -->
+   <jvmreuse>false</jvmreuse> 
+
+   <!-- default block dim for binary block files -->
+   <defaultblocksize>1000</defaultblocksize> 
+
+   <!-- run systemml control program as yarn appmaster, in case of MR1 always 
falls back to client, please disable for debug mode -->
+   <dml.yarn.appmaster>false</dml.yarn.appmaster>
+
+   <!-- maximum jvm heap size of the dml yarn appmaster in MB, the requested 
memory is 1.5x this parameter -->
+   <dml.yarn.appmaster.mem>2048</dml.yarn.appmaster.mem>
+
+   <!-- maximum jvm heap size of the map/reduce tasks in MB, the requested 
memory is 1.5x this parameter, negative values ignored  -->
+   <dml.yarn.mapreduce.mem>2048</dml.yarn.mapreduce.mem>
+
+   <!-- yarn application submission queue, relevant for default capacity 
scheduler -->
+   <dml.yarn.app.queue>default</dml.yarn.app.queue>
+   
+   <!-- enables multi-threaded matrix multiplications in singlenode control 
program -->
+   <cp.parallel.matrixmult>true</cp.parallel.matrixmult>
+   
+   <!-- enables multi-threaded read/write of text formats in singlenode 
control program -->
+   <cp.parallel.textio>true</cp.parallel.textio>
+   
+   <!-- enables automatic code generation -->
+   <codegen.enabled>true</codegen.enabled>
+   <codegen.plancache>true</codegen.plancache>
+   <codegen.literals>1</codegen.literals>
+</root>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/test/scripts/functions/codegen/cellwisetmpl9.R
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/cellwisetmpl9.R 
b/src/test/scripts/functions/codegen/cellwisetmpl9.R
new file mode 100644
index 0000000..5d25389
--- /dev/null
+++ b/src/test/scripts/functions/codegen/cellwisetmpl9.R
@@ -0,0 +1,32 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+args<-commandArgs(TRUE)
+options(digits=22)
+library("Matrix")
+
+X = matrix(seq(7, 1006), 500, 2);
+Y = matrix(seq(6, 1005), 500, 2);
+
+Z = X + 7 * Y;
+R = sum(Z^2)
+
+write(R, paste(args[2],"S",sep=""))

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d33537a9/src/test/scripts/functions/codegen/cellwisetmpl9.dml
----------------------------------------------------------------------
diff --git a/src/test/scripts/functions/codegen/cellwisetmpl9.dml 
b/src/test/scripts/functions/codegen/cellwisetmpl9.dml
new file mode 100644
index 0000000..b86a086
--- /dev/null
+++ b/src/test/scripts/functions/codegen/cellwisetmpl9.dml
@@ -0,0 +1,28 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+X = matrix(seq(7, 1006), 500, 2);
+Y = matrix(seq(6, 1005), 500, 2);
+
+Z = X + 7 * Y;
+R = sum(Z^2)
+
+write(R, $1)

Reply via email to