[18/23] systemml git commit: Get rid of leftover Guava dependency

[mboehm7]

Get rid of leftover Guava dependency


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/c4e9228e
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/c4e9228e
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/c4e9228e

Branch: refs/heads/master
Commit: c4e9228ed0b86789f8b41a533bf112d681a30318
Parents: 3c4d777
Author: Dylan Hutchison 
Authored: Tue Jul 11 22:46:33 2017 -0700
Committer: Dylan Hutchison 
Committed: Tue Jul 11 22:46:33 2017 -0700

--
 ...RewriteElementwiseMultChainOptimization.java | 32 +++-
 1 file changed, 17 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/c4e9228e/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
--
diff --git 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
index 486072b..de1def8 100644
--- 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
+++ 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
@@ -21,6 +21,7 @@ package org.apache.sysml.hops.rewrite;
 
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
@@ -34,15 +35,12 @@ import org.apache.sysml.hops.HopsException;
 import org.apache.sysml.hops.LiteralOp;
 import org.apache.sysml.parser.Expression;
 
-import com.google.common.collect.HashMultiset;
-import com.google.common.collect.Multiset;
-
 /**
  * Prerequisite: RewriteCommonSubexpressionElimination must run before this 
rule.
  *
  * Rewrite a chain of element-wise multiply hops that contain identical 
elements.
  * For example `(B * A) * B` is rewritten to `A * (B^2)` (or `(B^2) * A`), 
where `^` is element-wise power.
- * The order of the multiplicands depends on their data types, dimentions 
(matrix or vector), and sparsity.
+ * The order of the multiplicands depends on their data types, dimensions 
(matrix or vector), and sparsity.
  *
  * Does not rewrite in the presence of foreign parents in the middle of the 
e-wise multiply chain,
  * since foreign parents may rely on the individual results.
@@ -87,7 +85,7 @@ public class RewriteElementwiseMultChainOptimization extends 
HopRewriteRule {
if (isBinaryMult(root)) {
final Hop left = root.getInput().get(0), right = 
root.getInput().get(1);
final Set emults = new HashSet<>();
-   final Multiset leaves = HashMultiset.create();
+   final Map leaves = new HashMap<>(); // 
poor man's HashMultiset
findEMultsAndLeaves((BinaryOp)root, emults, leaves);
 
// 2. Ensure it is profitable to do a rewrite.
@@ -109,7 +107,7 @@ public class RewriteElementwiseMultChainOptimization 
extends HopRewriteRule {
final Hop newRoot = 
HopRewriteUtils.rewireAllParentChildReferences(root, replacement);
 
// 6. Recurse at leaves (no need to 
repeat the interior emults)
-   for (final Hop leaf : 
leaves.elementSet()) {
+   for (final Hop leaf : leaves.keySet()) {
recurseInputs(leaf);
}
return newRoot;
@@ -131,15 +129,15 @@ public class RewriteElementwiseMultChainOptimization 
extends HopRewriteRule {
}
}
 
-   private static Hop constructReplacement(final Set emults, 
final Multiset leaves) {
+   private static Hop constructReplacement(final Set emults, 
final Map leaves) {
// Sort by data type
final SortedMap sorted = new 
TreeMap<>(compareByDataType);
-   for (final Multiset.Entry entry : leaves.entrySet()) {
-   final Hop h = entry.getElement();
+   for (final Map.Entry entry : leaves.entrySet()) {
+   final Hop h = entry.getKey();
// unlink parents that are in the emult set(we are 
throwing them away)
// keep other parents
h.getParent().removeIf(parent -> parent instanceof 
BinaryOp && emults.contains(parent));
-   sorted.put(h, entry.getCount());
+   sorted.put(h, 

[05/23] systemml git commit: TernaryAggregate now applies to a power of 3.

[mboehm7]

TernaryAggregate now applies to a power of 3.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/f005d949
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/f005d949
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/f005d949

Branch: refs/heads/master
Commit: f005d94997d9c17ad8e90b4d2bd340f81b9a752d
Parents: 8b832f6
Author: Dylan Hutchison 
Authored: Fri Jun 9 22:06:10 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:24 2017 -0700

--
 .../java/org/apache/sysml/hops/AggUnaryOp.java  | 67 
 .../functions/misc/RewriteEMultChainTest.java   |  7 +-
 .../functions/misc/RewriteEMultChainOp.R| 33 --
 .../functions/misc/RewriteEMultChainOp.dml  | 28 
 .../functions/misc/RewriteEMultChainOpXYX.R | 33 ++
 .../functions/misc/RewriteEMultChainOpXYX.dml   | 28 
 6 files changed, 106 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/f005d949/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
--
diff --git a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java 
b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
index 4573b66..300a20c 100644
--- a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
@@ -490,29 +490,35 @@ public class AggUnaryOp extends Hop implements 
MultiThreadedHop
(_direction == Direction.RowCol || _direction == 
Direction.Col)  ) 
{
Hop input1 = getInput().get(0);
-   if( input1.getParent().size() == 1 && //sum single 
consumer
-   input1 instanceof BinaryOp && 
((BinaryOp)input1).getOp()==OpOp2.MULT
-   // As unary agg instruction is not implemented 
in MR and since MR is in maintenance mode, postponed it.
-   && input1.optFindExecType() != ExecType.MR) 
-   {
-   Hop input11 = input1.getInput().get(0);
-   Hop input12 = input1.getInput().get(1);
-   
-   if( input11 instanceof BinaryOp && 
((BinaryOp)input11).getOp()==OpOp2.MULT ) {
-   //ternary, arbitrary matrices but no 
mv/outer operations.
-   ret = 
HopRewriteUtils.isEqualSize(input11.getInput().get(0), input1)
-   && 
HopRewriteUtils.isEqualSize(input11.getInput().get(1), input1)   
-   && 
HopRewriteUtils.isEqualSize(input12, input1);
-   }
-   else if( input12 instanceof BinaryOp && 
((BinaryOp)input12).getOp()==OpOp2.MULT ) {
-   //ternary, arbitrary matrices but no 
mv/outer operations.
-   ret = 
HopRewriteUtils.isEqualSize(input12.getInput().get(0), input1)
-   && 
HopRewriteUtils.isEqualSize(input12.getInput().get(1), input1)   
-   && 
HopRewriteUtils.isEqualSize(input11, input1);
+   if (input1.getParent().size() == 1
+   && input1 instanceof BinaryOp) { //sum 
single consumer
+   BinaryOp binput1 = (BinaryOp)input1;
+
+   if (binput1.getOp() == OpOp2.POW
+   && binput1.getInput().get(1) 
instanceof LiteralOp) {
+   LiteralOp lit = 
(LiteralOp)binput1.getInput().get(1);
+   ret = lit.getLongValue() == 3;
}
-   else {
-   //binary, arbitrary matrices but no 
mv/outer operations.
-   ret = 
HopRewriteUtils.isEqualSize(input11, input12);
+   else if (binput1.getOp() == OpOp2.MULT
+   // As unary agg instruction is 
not implemented in MR and since MR is in maintenance mode, postponed it.
+   && input1.optFindExecType() != 
ExecType.MR) {
+   Hop input11 = input1.getInput().get(0);
+   Hop input12 = input1.getInput().get(1);
+
+   if (input11 

[02/23] systemml git commit: Fix RewriteEMult comparator. Add tests.

[mboehm7]

Fix RewriteEMult comparator. Add tests.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/eb0599df
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/eb0599df
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/eb0599df

Branch: refs/heads/master
Commit: eb0599df4c3bcca15531b85a3d870a26e4653179
Parents: 7d57883
Author: Dylan Hutchison 
Authored: Fri Jun 9 11:18:32 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:15 2017 -0700

--
 .../org/apache/sysml/hops/OptimizerUtils.java   |   9 +-
 .../sysml/hops/rewrite/ProgramRewriter.java |   3 +-
 .../apache/sysml/hops/rewrite/RewriteEMult.java |  10 +-
 .../functions/misc/RewriteEMultChainTest.java   | 127 +
 .../ternary/ABATernaryAggregateTest.java| 268 +++
 .../functions/misc/RewriteEMultChainOp.R|  33 +++
 .../functions/misc/RewriteEMultChainOp.dml  |  28 ++
 .../functions/ternary/ABATernaryAggregateC.R|  32 +++
 .../functions/ternary/ABATernaryAggregateC.dml  |  30 +++
 .../functions/ternary/ABATernaryAggregateRC.R   |  33 +++
 .../functions/ternary/ABATernaryAggregateRC.dml |  30 +++
 11 files changed, 597 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/eb0599df/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
--
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java 
b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index a40e36c..2a76d07 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -110,8 +110,13 @@ public class OptimizerUtils
 */
public static boolean ALLOW_CONSTANT_FOLDING = true;

-   public static boolean ALLOW_ALGEBRAIC_SIMPLIFICATION = true; 
-   public static boolean ALLOW_OPERATOR_FUSION = true; 
+   public static boolean ALLOW_ALGEBRAIC_SIMPLIFICATION = true;
+   /**
+* Enables rewriting chains of element-wise multiplies that contain the 
same multiplicand more than once, as in
+* `A*B*A ==> (A^2)*B`.
+*/
+   public static boolean ALLOW_EMULT_CHAIN_REWRITE = true;
+   public static boolean ALLOW_OPERATOR_FUSION = true;

/**
 * Enables if-else branch removal for constant predicates (original 
literals or 

http://git-wip-us.apache.org/repos/asf/systemml/blob/eb0599df/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java 
b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
index 8573dd7..b6aab38 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
@@ -96,7 +96,8 @@ public class ProgramRewriter
_dagRuleSet.add( new 
RewriteRemoveUnnecessaryCasts() ); 
if( 
OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION )
_dagRuleSet.add( new 
RewriteCommonSubexpressionElimination() );
-   _dagRuleSet.add( new RewriteEMult() 
 ); //dependency: cse
+   if ( OptimizerUtils.ALLOW_EMULT_CHAIN_REWRITE )
+   _dagRuleSet.add( new RewriteEMult() 
 ); //dependency: cse
if( OptimizerUtils.ALLOW_CONSTANT_FOLDING )
_dagRuleSet.add( new RewriteConstantFolding()   
 ); //dependency: cse
if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION )

http://git-wip-us.apache.org/repos/asf/systemml/blob/eb0599df/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
index 47c32a9..2c9e5cb 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
@@ -50,7 +50,6 @@ public class RewriteEMult extends HopRewriteRule {
public ArrayList rewriteHopDAGs(ArrayList roots, 
ProgramRewriteStatus state) throws HopsException {
if( roots == null )
return null;
-
for( int i=0; i

[19/23] systemml git commit: Move to dynamic rewrites. Do not rewrite if top-level dims unknown.

[mboehm7]

Move to dynamic rewrites. Do not rewrite if top-level dims unknown.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d18a4c80
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d18a4c80
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d18a4c80

Branch: refs/heads/master
Commit: d18a4c80dece566ddbad34a7f3c2f70ce544023e
Parents: c4e9228
Author: Dylan Hutchison 
Authored: Tue Jul 11 22:54:31 2017 -0700
Committer: Dylan Hutchison 
Committed: Tue Jul 11 22:54:31 2017 -0700

--
 .../java/org/apache/sysml/hops/rewrite/ProgramRewriter.java| 6 +++---
 .../hops/rewrite/RewriteElementwiseMultChainOptimization.java  | 5 +++--
 2 files changed, 6 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/d18a4c80/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java 
b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
index a1ff5bc..59565df 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
@@ -96,8 +96,6 @@ public class ProgramRewriter
_dagRuleSet.add( new 
RewriteRemoveUnnecessaryCasts() ); 
if( 
OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION )
_dagRuleSet.add( new 
RewriteCommonSubexpressionElimination() );
-   if ( OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES)
-   _dagRuleSet.add( new 
RewriteElementwiseMultChainOptimization()   ); //dependency: cse
if( OptimizerUtils.ALLOW_CONSTANT_FOLDING )
_dagRuleSet.add( new RewriteConstantFolding()   
 ); //dependency: cse
if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION )
@@ -125,7 +123,9 @@ public class ProgramRewriter
// DYNAMIC REWRITES (which do require size information)
if( dynamicRewrites )
{
-   _dagRuleSet.add( new 
RewriteMatrixMultChainOptimization() ); //dependency: cse 
+   _dagRuleSet.add( new 
RewriteMatrixMultChainOptimization() ); //dependency: cse
+   if ( OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES)
+   _dagRuleSet.add( new 
RewriteElementwiseMultChainOptimization()); //dependency: cse

if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION )
{

http://git-wip-us.apache.org/repos/asf/systemml/blob/d18a4c80/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
--
diff --git 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
index de1def8..1f85bbf 100644
--- 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
+++ 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
@@ -44,6 +44,7 @@ import org.apache.sysml.parser.Expression;
  *
  * Does not rewrite in the presence of foreign parents in the middle of the 
e-wise multiply chain,
  * since foreign parents may rely on the individual results.
+ * Does not perform rewrites on an element-wise multiply if its dimensions are 
unknown.
  *
  * The new order of element-wise multiply chains is as follows:
  * 
@@ -81,8 +82,8 @@ public class RewriteElementwiseMultChainOptimization extends 
HopRewriteRule {
return root;
root.setVisited();
 
-   // 1. Find immediate subtree of EMults.
-   if (isBinaryMult(root)) {
+   // 1. Find immediate subtree of EMults. Check dimsKnown.
+   if (isBinaryMult(root) && root.dimsKnown()) {
final Hop left = root.getInput().get(0), right = 
root.getInput().get(1);
final Set emults = new HashSet<>();
final Map leaves = new HashMap<>(); // 
poor man's HashMultiset

[07/23] systemml git commit: simplifyDotProductSum shall not interfere with tak+*

[mboehm7]

simplifyDotProductSum shall not interfere with tak+*

Added conditions to the dynamic algebraic rewrite simplifyDotProductSum
that do not apply the optimization for (A^2)*B or B*(A^2),
since TernaryAggregate handles these.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/a5846bbb
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/a5846bbb
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/a5846bbb

Branch: refs/heads/master
Commit: a5846bbb383c655189963bffefed1c0db4ffcc89
Parents: edbac3b
Author: Dylan Hutchison 
Authored: Fri Jun 9 23:58:16 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:30 2017 -0700

--
 .../hops/rewrite/RewriteAlgebraicSimplificationDynamic.java | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/a5846bbb/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
--
diff --git 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
index ad80c05..166af2f 100644
--- 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
+++ 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
@@ -2050,7 +2050,14 @@ public class RewriteAlgebraicSimplificationDynamic 
extends HopRewriteRule
else if( HopRewriteUtils.isBinary(hi2, OpOp2.MULT, 1) 
//no other consumer than sum
&& hi2.getInput().get(0).getDim2()==1 
&& hi2.getInput().get(1).getDim2()==1
&& 
!HopRewriteUtils.isBinary(hi2.getInput().get(0), OpOp2.MULT)
-   && 
!HopRewriteUtils.isBinary(hi2.getInput().get(1), OpOp2.MULT) )
+   && 
!HopRewriteUtils.isBinary(hi2.getInput().get(1), OpOp2.MULT)
+   && 
!(HopRewriteUtils.isBinary(hi2.getInput().get(0), OpOp2.POW)  // do not 
rewrite (A^2)*B
+   && 
hi2.getInput().get(0).getInput().get(1) instanceof LiteralOp  // let tak+* 
handle it
+   && 
((LiteralOp)hi2.getInput().get(0).getInput().get(1)).getLongValue() == 2)
+   && 
!(HopRewriteUtils.isBinary(hi2.getInput().get(1), OpOp2.POW)  // do not 
rewrite B*(A^2)
+   && 
hi2.getInput().get(1).getInput().get(1) instanceof LiteralOp  // let tak+* 
handle it
+   && 
((LiteralOp)hi2.getInput().get(1).getInput().get(1)).getLongValue() == 2)
+   )
{
baLeft = hi2.getInput().get(0);
baRight = hi2.getInput().get(1);

[17/23] systemml git commit: Merge branch 'rewrite-emult' into rewrite-emult2

[mboehm7]

Merge branch 'rewrite-emult' into rewrite-emult2

# Conflicts:
#   src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java
#   src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
#   
src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
#   
src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteElementwiseMultChainOptimizationTest.java


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/3c4d777a
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/3c4d777a
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/3c4d777a

Branch: refs/heads/master
Commit: 3c4d777a5a15ed59681547e91b84c8812d3420fc
Parents: b67f186 999fdfb
Author: Dylan Hutchison 
Authored: Tue Jul 11 22:26:59 2017 -0700
Committer: Dylan Hutchison 
Committed: Tue Jul 11 22:37:36 2017 -0700

--
 .../sysml/hops/rewrite/ProgramRewriter.java |   4 +-
 ...RewriteElementwiseMultChainOptimization.java | 184 +--
 ...ElementwiseMultChainOptimizationAllTest.java | 134 ++
 ...iteElementwiseMultChainOptimizationTest.java |   4 +-
 .../functions/misc/RewriteEMultChainOpAll.R |  37 
 .../functions/misc/RewriteEMultChainOpAll.dml   |  31 
 6 files changed, 334 insertions(+), 60 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/3c4d777a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
--

[15/23] systemml git commit: Change order of row and col vectors, so as to create inner products rather than outer products.

[mboehm7]

Change order of row and col vectors, so as to create inner products rather than 
outer products.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/6c3e1c5b
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/6c3e1c5b
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/6c3e1c5b

Branch: refs/heads/master
Commit: 6c3e1c5bad30dc8f11ff9d3f412ce68873c37202
Parents: 04f692d
Author: Dylan Hutchison 
Authored: Tue Jul 11 20:08:22 2017 -0700
Committer: Dylan Hutchison 
Committed: Tue Jul 11 20:08:22 2017 -0700

--
 ...RewriteElementwiseMultChainOptimization.java | 24 ++--
 1 file changed, 12 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/6c3e1c5b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
--
diff --git 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
index 9ca0932..9cc8fcd 100644
--- 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
+++ 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
@@ -162,7 +162,7 @@ public class RewriteElementwiseMultChainOptimization 
extends HopRewriteRule {
/**
 * A Comparator that orders Hops by their data type, dimention, and 
sparsity.
 * The order is as follows:
-*  scalars > row vectors > col vectors >
+*  scalars > col vectors > row vectors >
 *  non-vector matrices ordered by sparsity (higher nnz first, 
unknown sparsity last) >
 *  other data types.
 * Disambiguate by Hop ID.
@@ -181,23 +181,23 @@ public class RewriteElementwiseMultChainOptimization 
extends HopRewriteRule {
}
 
@Override
-   public final int compare(Hop o1, Hop o2) {
-   int c = 
Integer.compare(orderDataType[o1.getDataType().ordinal()], 
orderDataType[o2.getDataType().ordinal()]);
+   public final int compare(final Hop o1, final Hop o2) {
+   final int c = 
Integer.compare(orderDataType[o1.getDataType().ordinal()], 
orderDataType[o2.getDataType().ordinal()]);
if (c != 0) return c;
 
// o1 and o2 have the same data type
switch (o1.getDataType()) {
case MATRIX:
// two matrices; check for vectors
-   if (o1.getDim1() == 1) { // row vector
-   if (o2.getDim1() != 1) return 
1; // row vectors are greatest of matrices
-   return 
compareBySparsityThenId(o1, o2); // both row vectors
-   } else if (o2.getDim1() == 1) { // 2 is row 
vector; 1 is not
-   return -1; // row vectors are 
the greatest matrices
-   } else if (o1.getDim2() == 1) { // col vector
-   if (o2.getDim2() != 1) return 
1; // col vectors greater than non-vectors
+   if (o1.getDim2() == 1) { // col vector
+   if (o2.getDim2() != 1) return 
1; // col vectors are greatest of matrices
return 
compareBySparsityThenId(o1, o2); // both col vectors
} else if (o2.getDim2() == 1) { // 2 is col 
vector; 1 is not
+   return -1; // col vectors are 
the greatest matrices
+   } else if (o1.getDim1() == 1) { // row vector
+   if (o2.getDim1() != 1) return 
1; // row vectors greater than non-vectors
+   return 
compareBySparsityThenId(o1, o2); // both row vectors
+   } else if (o2.getDim1() == 1) { // 2 is row 
vector; 1 is not
return 1; // col vectors 
greater than non-vectors
} else { // both non-vectors
return 
compareBySparsityThenId(o1, o2);
@@ -206,9 +206,9 @@ public class RewriteElementwiseMultChainOptimization 
extends HopRewriteRule {
return Long.compare(o1.getHopID(), 
o2.getHopID());
}
}
-  

[08/23] systemml git commit: Document RewriteEMult. Add smart recursion.

[mboehm7]

Document RewriteEMult. Add smart recursion.

RewriteEMult now rewrites emult chains deeper than the top-most one.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d88f867f
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d88f867f
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d88f867f

Branch: refs/heads/master
Commit: d88f867fd0384954dce9e6ce4d65f02f1054bc5e
Parents: a5846bb
Author: Dylan Hutchison 
Authored: Sat Jun 10 01:17:36 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:33 2017 -0700

--
 .../sysml/hops/rewrite/HopRewriteUtils.java |  2 +
 .../apache/sysml/hops/rewrite/RewriteEMult.java | 90 +---
 .../org/apache/sysml/parser/Expression.java |  1 -
 3 files changed, 62 insertions(+), 31 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/d88f867f/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java 
b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
index 4d23cb9..17ac4ec 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
@@ -246,6 +246,8 @@ public class HopRewriteUtils
 * Replace an old Hop with a replacement Hop.
 * If the old Hop has no parents, then return the replacement.
 * Otherwise rewire each of the Hop's parents into the replacement and 
return the replacement.
+* @param old To be replaced
+* @param replacement The replacement
 * @return replacement
 */
public static Hop replaceHop(final Hop old, final Hop replacement) {

http://git-wip-us.apache.org/repos/asf/systemml/blob/d88f867f/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
index d483a08..5cd1471 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
@@ -42,6 +42,7 @@ import com.google.common.collect.Multiset;
  *
  * Rewrite a chain of element-wise multiply hops that contain identical 
elements.
  * For example `(B * A) * B` is rewritten to `A * (B^2)` (or `(B^2) * A`), 
where `^` is element-wise power.
+ * The order of the multiplicands depends on their data types, dimentions 
(matrix or vector), and sparsity.
  *
  * Does not rewrite in the presence of foreign parents in the middle of the 
e-wise multiply chain,
  * since foreign parents may rely on the individual results.
@@ -74,18 +75,15 @@ public class RewriteEMult extends HopRewriteRule {
return root;
root.setVisited();
 
-   final ArrayList rootInputs = root.getInput();
-
// 1. Find immediate subtree of EMults.
if (isBinaryMult(root)) {
-   final Hop left = rootInputs.get(0), right = 
rootInputs.get(1);
-   final BinaryOp r = (BinaryOp)root;
+   final Hop left = root.getInput().get(0), right = 
root.getInput().get(1);
final Set emults = new HashSet<>();
final Multiset leaves = HashMultiset.create();
-   findEMultsAndLeaves(r, emults, leaves);
+   findEMultsAndLeaves((BinaryOp)root, emults, leaves);
 
// 2. Ensure it is profitable to do a rewrite.
-   if (isOptimizable(leaves)) {
+   if (isOptimizable(emults, leaves)) {
// 3. Check for foreign parents.
// A foreign parent is a parent of some EMult 
that is not in the set.
// Foreign parents destroy correctness of this 
rewrite.
@@ -94,25 +92,35 @@ public class RewriteEMult extends HopRewriteRule {
if (okay) {
// 4. Construct replacement EMults for 
the leaves
final Hop replacement = 
constructReplacement(leaves);
-   // 5. Replace root with replacement
if (LOG.isDebugEnabled())
LOG.debug(String.format(
"Element-wise 
multiply chain rewrite of %d e-mults at 

[01/23] systemml git commit: New rewrite rule for chains of element-wise multiply.

[mboehm7]

Repository: systemml
Updated Branches:
  refs/heads/master 1b3dff06b -> 85e3a9631


New rewrite rule for chains of element-wise multiply.

Placed rewrite rule after Common Subexpression Elimination.
Included helper method in HopRewriteUtils.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/7d578838
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/7d578838
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/7d578838

Branch: refs/heads/master
Commit: 7d578838cc291a1adb6229bae01f7c9428b6f858
Parents: c434208
Author: Dylan Hutchison 
Authored: Thu Jun 8 18:17:36 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:13 2017 -0700

--
 .../sysml/hops/rewrite/HopRewriteUtils.java |  17 +-
 .../sysml/hops/rewrite/ProgramRewriter.java |   1 +
 .../apache/sysml/hops/rewrite/RewriteEMult.java | 186 +++
 .../org/apache/sysml/parser/Expression.java |   1 +
 4 files changed, 204 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/7d578838/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java 
b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
index cf6081b..4d23cb9 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
@@ -241,7 +241,22 @@ public class HopRewriteUtils
parent.getInput().add( pos, child );
child.getParent().add( parent );
}
-   
+
+   /**
+* Replace an old Hop with a replacement Hop.
+* If the old Hop has no parents, then return the replacement.
+* Otherwise rewire each of the Hop's parents into the replacement and 
return the replacement.
+* @return replacement
+*/
+   public static Hop replaceHop(final Hop old, final Hop replacement) {
+   final ArrayList rootParents = old.getParent();
+   if (rootParents.isEmpty())
+   return replacement; // new old!
+   HopRewriteUtils.rewireAllParentChildReferences(old, 
replacement);
+   return replacement;
+   }
+
+
public static void rewireAllParentChildReferences( Hop hold, Hop hnew ) 
{
ArrayList parents = new ArrayList(hold.getParent());
for( Hop lparent : parents )

http://git-wip-us.apache.org/repos/asf/systemml/blob/7d578838/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java 
b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
index 0e65f3f..8573dd7 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
@@ -96,6 +96,7 @@ public class ProgramRewriter
_dagRuleSet.add( new 
RewriteRemoveUnnecessaryCasts() ); 
if( 
OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION )
_dagRuleSet.add( new 
RewriteCommonSubexpressionElimination() );
+   _dagRuleSet.add( new RewriteEMult() 
 ); //dependency: cse
if( OptimizerUtils.ALLOW_CONSTANT_FOLDING )
_dagRuleSet.add( new RewriteConstantFolding()   
 ); //dependency: cse
if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION )

http://git-wip-us.apache.org/repos/asf/systemml/blob/7d578838/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
new file mode 100644
index 000..47c32a9
--- /dev/null
+++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteEMult.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless 

[12/23] systemml git commit: Fix visit status bug

[mboehm7]

Fix visit status bug


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/0a8936cd
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/0a8936cd
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/0a8936cd

Branch: refs/heads/master
Commit: 0a8936cd849d74baced732f45f1c53812abce537
Parents: d6d3795
Author: Dylan Hutchison 
Authored: Sun Jun 11 03:55:25 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:48 2017 -0700

--
 .../apache/sysml/hops/rewrite/HopDagValidator.java |  5 -
 .../RewriteElementwiseMultChainOptimization.java   | 17 +
 2 files changed, 13 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/0a8936cd/src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java 
b/src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java
index 8cb5e1e..9ac21fc 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopDagValidator.java
@@ -35,6 +35,8 @@ import org.apache.sysml.parser.Expression;
 import org.apache.sysml.runtime.DMLRuntimeException;
 import org.apache.sysml.utils.Explain;
 
+import com.google.common.collect.Lists;
+
 import static org.apache.sysml.hops.HopsException.check;
 
 /**
@@ -89,7 +91,8 @@ public class HopDagValidator {
//check visit status
final boolean seen = !state.seen.add(id);
check(seen == hop.isVisited(), hop,
-   "seen previously is %b but does not match hop 
visit status", seen);
+   "(parents: %s) seen previously is %b but does 
not match hop visit status",
+   Lists.transform(hop.getParent(), 
Hop::getHopID), seen);
if (seen) return; // we saw the Hop previously, no need to 
re-validate

//check parent linking

http://git-wip-us.apache.org/repos/asf/systemml/blob/0a8936cd/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
--
diff --git 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
index 91b7306..9ca0932 100644
--- 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
+++ 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
@@ -91,7 +91,7 @@ public class RewriteElementwiseMultChainOptimization extends 
HopRewriteRule {
(!isBinaryMult(right) || 
checkForeignParent(emults, (BinaryOp)right));
if (okay) {
// 4. Construct replacement EMults for 
the leaves
-   final Hop replacement = 
constructReplacement(leaves);
+   final Hop replacement = 
constructReplacement(emults, leaves);
if (LOG.isDebugEnabled())
LOG.debug(String.format(
"Element-wise 
multiply chain rewrite of %d e-mults at sub-dag %d to new sub-dag %d",
@@ -123,13 +123,14 @@ public class RewriteElementwiseMultChainOptimization 
extends HopRewriteRule {
}
}
 
-   private static Hop constructReplacement(final Multiset leaves) {
+   private static Hop constructReplacement(final Set emults, 
final Multiset leaves) {
// Sort by data type
final SortedMap sorted = new 
TreeMap<>(compareByDataType);
for (final Multiset.Entry entry : leaves.entrySet()) {
final Hop h = entry.getElement();
-   // unlink parents (the EMults, which we are throwing 
away)
-   h.getParent().clear();
+   // unlink parents that are in the emult set(we are 
throwing them away)
+   // keep other parents
+   h.getParent().removeIf(parent -> parent instanceof 
BinaryOp && emults.contains(parent));
sorted.put(h, entry.getCount());
}
// sorted contains all leaves, sorted by data type, stripped 
from their parents
@@ -146,12 +147,13 @@ public class RewriteElementwiseMultChainOptimization 

[09/23] systemml git commit: Review comments, part 1

[mboehm7]

Review comments, part 1


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/b94557fd
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/b94557fd
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/b94557fd

Branch: refs/heads/master
Commit: b94557fd2c90c591179cdbf05a32242fadc36448
Parents: d88f867
Author: Dylan Hutchison 
Authored: Sun Jun 11 00:35:52 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:37 2017 -0700

--
 .../java/org/apache/sysml/hops/AggUnaryOp.java  |  21 +-
 .../org/apache/sysml/hops/OptimizerUtils.java   |   5 -
 .../sysml/hops/rewrite/HopRewriteUtils.java |   3 +-
 .../sysml/hops/rewrite/ProgramRewriter.java |   4 +-
 .../RewriteAlgebraicSimplificationDynamic.java  |  16 +-
 .../apache/sysml/hops/rewrite/RewriteEMult.java | 284 ---
 ...RewriteElementwiseMultChainOptimization.java | 281 ++
 .../functions/misc/RewriteEMultChainTest.java   | 127 -
 ...ementwiseMultChainOptimizationChainTest.java | 127 +
 .../ternary/ABATernaryAggregateTest.java|   9 +-
 .../functions/misc/ZPackageSuite.java   |   1 +
 .../functions/ternary/ZPackageSuite.java|   3 +-
 12 files changed, 436 insertions(+), 445 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/b94557fd/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
--
diff --git a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java 
b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
index 300a20c..a207831 100644
--- a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
@@ -497,7 +497,7 @@ public class AggUnaryOp extends Hop implements 
MultiThreadedHop
if (binput1.getOp() == OpOp2.POW
&& binput1.getInput().get(1) 
instanceof LiteralOp) {
LiteralOp lit = 
(LiteralOp)binput1.getInput().get(1);
-   ret = lit.getLongValue() == 3;
+   ret = 
HopRewriteUtils.getIntValueSafe(lit) == 3;
}
else if (binput1.getOp() == OpOp2.MULT
// As unary agg instruction is 
not implemented in MR and since MR is in maintenance mode, postponed it.
@@ -640,15 +640,10 @@ public class AggUnaryOp extends Hop implements 
MultiThreadedHop
boolean handled = false;
 
if (input1.getOp() == OpOp2.POW) {
-   switch ((int)((LiteralOp)input12).getLongValue()) {
-   case 3:
-   in1 = input11.constructLops();
-   in2 = in1;
-   in3 = in1;
-   break;
-   default:
-   throw new AssertionError("unreachable; only 
applies to power 3");
-   }
+   assert(HopRewriteUtils.isLiteralOfValue(input12, 3)) : 
"this case can only occur with a power of 3";
+   in1 = input11.constructLops();
+   in2 = in1;
+   in3 = in1;
handled = true;
} else if (input11 instanceof BinaryOp ) {
BinaryOp b11 = (BinaryOp)input11;
@@ -662,8 +657,7 @@ public class AggUnaryOp extends Hop implements 
MultiThreadedHop
case POW: // A*A*B case
Hop b112 = b11.getInput().get(1);
if ( !(input12 instanceof BinaryOp && 
((BinaryOp)input12).getOp()==OpOp2.MULT)
-   && b112 instanceof LiteralOp
-   && 
((LiteralOp)b112).getLongValue() == 2) {
+   && 
HopRewriteUtils.isLiteralOfValue(b112, 2) ) {
in1 = 
b11.getInput().get(0).constructLops();
in2 = in1;
in3 = input12.constructLops();
@@ -682,8 +676,7 @@ public class AggUnaryOp extends Hop implements 
MultiThreadedHop
break;
case POW: // A*B*B case
Hop b112 = b12.getInput().get(1);
-   if ( b112 instanceof LiteralOp
-   && 
((LiteralOp)b112).getLongValue() == 2) {
+   if 

[14/23] systemml git commit: Review comments 3

[mboehm7]

Review comments 3


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/04f692df
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/04f692df
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/04f692df

Branch: refs/heads/master
Commit: 04f692dfcb25a032044dabb7064241073f959300
Parents: de469d2
Author: Dylan Hutchison 
Authored: Sun Jun 18 16:54:51 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:54 2017 -0700

--
 .../java/org/apache/sysml/hops/AggUnaryOp.java  |   4 +-
 .../sysml/hops/rewrite/ProgramRewriter.java |   2 +-
 ...ementwiseMultChainOptimizationChainTest.java | 127 ---
 ...iteElementwiseMultChainOptimizationTest.java | 127 +++
 .../functions/misc/ZPackageSuite.java   |   2 +-
 5 files changed, 132 insertions(+), 130 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/04f692df/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
--
diff --git a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java 
b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
index a207831..8e681c1 100644
--- a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
@@ -647,7 +647,7 @@ public class AggUnaryOp extends Hop implements 
MultiThreadedHop
handled = true;
} else if (input11 instanceof BinaryOp ) {
BinaryOp b11 = (BinaryOp)input11;
-   switch (b11.getOp()) {
+   switch( b11.getOp() ) {
case MULT: // A*B*C case
in1 = input11.getInput().get(0).constructLops();
in2 = input11.getInput().get(1).constructLops();
@@ -664,6 +664,7 @@ public class AggUnaryOp extends Hop implements 
MultiThreadedHop
handled = true;
}
break;
+   default: break;
}
} else if( input12 instanceof BinaryOp ) {
BinaryOp b12 = (BinaryOp)input12;
@@ -683,6 +684,7 @@ public class AggUnaryOp extends Hop implements 
MultiThreadedHop
handled = true;
}
break;
+   default: break;
}
}
 

http://git-wip-us.apache.org/repos/asf/systemml/blob/04f692df/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java 
b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
index 1053850..7ee3ccb 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
@@ -97,7 +97,7 @@ public class ProgramRewriter
if( 
OptimizerUtils.ALLOW_COMMON_SUBEXPRESSION_ELIMINATION )
_dagRuleSet.add( new 
RewriteCommonSubexpressionElimination() );
if ( OptimizerUtils.ALLOW_SUM_PRODUCT_REWRITES)
-   _dagRuleSet.add( new 
RewriteElementwiseMultChainOptimization()  ); 
//dependency: cse
+   _dagRuleSet.add( new 
RewriteElementwiseMultChainOptimization()   ); //dependency: cse
if( OptimizerUtils.ALLOW_CONSTANT_FOLDING )
_dagRuleSet.add( new RewriteConstantFolding()   
 ); //dependency: cse
if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION )

http://git-wip-us.apache.org/repos/asf/systemml/blob/04f692df/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteElementwiseMultChainOptimizationChainTest.java
--
diff --git 
a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteElementwiseMultChainOptimizationChainTest.java
 
b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteElementwiseMultChainOptimizationChainTest.java
deleted file mode 100644
index e490750..000
--- 
a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteElementwiseMultChainOptimizationChainTest.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * 

[10/23] systemml git commit: Add scalars to Rewrite Emult test

[mboehm7]

Add scalars to Rewrite Emult test

Not sure how to check this in an assert statement


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/737f93b1
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/737f93b1
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/737f93b1

Branch: refs/heads/master
Commit: 737f93b15a96aba31bc6c6da3651be309e3b8b0c
Parents: b94557f
Author: Dylan Hutchison 
Authored: Sun Jun 11 01:56:07 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:41 2017 -0700

--
 .../hops/rewrite/RewriteElementwiseMultChainOptimization.java   | 2 +-
 src/main/java/org/apache/sysml/utils/Explain.java   | 4 ++--
 .../misc/RewriteElementwiseMultChainOptimizationChainTest.java  | 4 ++--
 .../integration/functions/ternary/ABATernaryAggregateTest.java  | 5 +
 src/test/scripts/functions/misc/RewriteEMultChainOpXYX.R| 4 ++--
 src/test/scripts/functions/misc/RewriteEMultChainOpXYX.dml  | 2 +-
 6 files changed, 9 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/737f93b1/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
--
diff --git 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
index bd873ff..1dd5813 100644
--- 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
+++ 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteElementwiseMultChainOptimization.java
@@ -222,7 +222,7 @@ public class RewriteElementwiseMultChainOptimization 
extends HopRewriteRule {
}
}
}
-   };
+   }.reversed();
 
/**
 * Check if a node has a parent that is not in the set of emults. 
Recursively check children who are also emults.

http://git-wip-us.apache.org/repos/asf/systemml/blob/737f93b1/src/main/java/org/apache/sysml/utils/Explain.java
--
diff --git a/src/main/java/org/apache/sysml/utils/Explain.java 
b/src/main/java/org/apache/sysml/utils/Explain.java
index 5cf0548..450c6e5 100644
--- a/src/main/java/org/apache/sysml/utils/Explain.java
+++ b/src/main/java/org/apache/sysml/utils/Explain.java
@@ -76,7 +76,7 @@ public class Explain
//internal configuration parameters
private static final boolean REPLACE_SPECIAL_CHARACTERS = true; 
private static final boolean SHOW_MEM_ABOVE_BUDGET  = true;
-   private static final boolean SHOW_LITERAL_HOPS  = false;
+   private static final boolean SHOW_LITERAL_HOPS  = true;
private static final boolean SHOW_DATA_DEPENDENCIES = true;
private static final boolean SHOW_DATA_FLOW_PROPERTIES  = true;
 
@@ -566,7 +566,7 @@ public class Explain
childs.append(" (");
boolean childAdded = false;
for( Hop input : hop.getInput() )
-   if( !(input instanceof LiteralOp) ){
+   if( SHOW_LITERAL_HOPS || !(input instanceof 
LiteralOp) ){
childs.append(childAdded?",":"");
childs.append(input.getHopID());
childAdded = true;

http://git-wip-us.apache.org/repos/asf/systemml/blob/737f93b1/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteElementwiseMultChainOptimizationChainTest.java
--
diff --git 
a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteElementwiseMultChainOptimizationChainTest.java
 
b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteElementwiseMultChainOptimizationChainTest.java
index 47b2f0e..e490750 100644
--- 
a/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteElementwiseMultChainOptimizationChainTest.java
+++ 
b/src/test/java/org/apache/sysml/test/integration/functions/misc/RewriteElementwiseMultChainOptimizationChainTest.java
@@ -33,7 +33,7 @@ import org.junit.Assert;
 import org.junit.Test;
 
 /**
- * Test whether `A*B*A` successfully rewrites to `(A^2)*B`.
+ * Test whether `2*X*3*Y*4*X` successfully rewrites to `Y*(X^2)*24`.
  */
 public class RewriteElementwiseMultChainOptimizationChainTest extends 
AutomatedTestBase
 {
@@ -96,7 +96,7 @@ public class RewriteElementwiseMultChainOptimizationChainTest 
extends AutomatedT

[21/23] systemml git commit: Add new `wumm` pattern to pick up element-wise multiply rewrite.

[mboehm7]

Add new `wumm` pattern to pick up element-wise multiply rewrite.

The new pattern recognizes when there is a `*2` or `2*` outside `W*(U%*%t(V))`.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/479b9da4
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/479b9da4
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/479b9da4

Branch: refs/heads/master
Commit: 479b9da4e6c605871a914ccb4b06ab6da5de21ed
Parents: e93c487
Author: Dylan Hutchison 
Authored: Thu Jul 13 01:14:48 2017 -0700
Committer: Dylan Hutchison 
Committed: Thu Jul 13 01:14:48 2017 -0700

--
 .../sysml/hops/rewrite/ProgramRewriter.java |  2 +-
 .../RewriteAlgebraicSimplificationDynamic.java  | 44 +++-
 2 files changed, 43 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/479b9da4/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java 
b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
index 59565df..7c4f861 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/ProgramRewriter.java
@@ -54,7 +54,7 @@ public class ProgramRewriter
private static final Log LOG = 
LogFactory.getLog(ProgramRewriter.class.getName());

//internal local debug level
-   private static final boolean LDEBUG = false; 
+   private static final boolean LDEBUG = false;
private static final boolean CHECK = false;

private ArrayList _dagRuleSet = null;

http://git-wip-us.apache.org/repos/asf/systemml/blob/479b9da4/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
--
diff --git 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
index 8cd71f4..6246270 100644
--- 
a/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
+++ 
b/src/main/java/org/apache/sysml/hops/rewrite/RewriteAlgebraicSimplificationDynamic.java
@@ -29,11 +29,11 @@ import org.apache.sysml.hops.AggUnaryOp;
 import org.apache.sysml.hops.BinaryOp;
 import org.apache.sysml.hops.DataGenOp;
 import org.apache.sysml.hops.Hop;
-import org.apache.sysml.hops.QuaternaryOp;
 import org.apache.sysml.hops.Hop.AggOp;
 import org.apache.sysml.hops.Hop.DataGenMethod;
 import org.apache.sysml.hops.Hop.Direction;
 import org.apache.sysml.hops.Hop.OpOp1;
+import org.apache.sysml.hops.Hop.OpOp2;
 import org.apache.sysml.hops.Hop.OpOp3;
 import org.apache.sysml.hops.Hop.OpOp4;
 import org.apache.sysml.hops.Hop.ParamBuiltinOp;
@@ -44,7 +44,7 @@ import org.apache.sysml.hops.LeftIndexingOp;
 import org.apache.sysml.hops.LiteralOp;
 import org.apache.sysml.hops.OptimizerUtils;
 import org.apache.sysml.hops.ParameterizedBuiltinOp;
-import org.apache.sysml.hops.Hop.OpOp2;
+import org.apache.sysml.hops.QuaternaryOp;
 import org.apache.sysml.hops.ReorgOp;
 import org.apache.sysml.hops.TernaryOp;
 import org.apache.sysml.hops.UnaryOp;
@@ -1959,6 +1959,46 @@ public class RewriteAlgebraicSimplificationDynamic 
extends HopRewriteRule
appliedPattern = true;
LOG.debug("Applied simplifyWeightedUnaryMM1 (line 
"+hi.getBeginLine()+")"); 
}
+
+   //Pattern 1.5) (W*(U%*%t(V))*2 or 2*(W*(U%*%t(V))
+   if( !appliedPattern
+   && hi instanceof BinaryOp && 
HopRewriteUtils.isValidOp(((BinaryOp)hi).getOp(), OpOp2.MULT)
+   && 
(HopRewriteUtils.isLiteralOfValue(hi.getInput().get(0), 2)
+   || 
HopRewriteUtils.isLiteralOfValue(hi.getInput().get(1), 2)))
+   {
+   final Hop nl; // non-literal
+   if( hi.getInput().get(0) instanceof LiteralOp ) {
+   nl = hi.getInput().get(1);
+   } else {
+   nl = hi.getInput().get(0);
+   }
+
+   if (   HopRewriteUtils.isBinary(nl, OpOp2.MULT)
+   && 
HopRewriteUtils.isEqualSize(nl.getInput().get(0), nl.getInput().get(1)) 
//prevent mv
+   && nl.getDim2() > 1 //not applied for 
vector-vector mult
+   && nl.getInput().get(0).getDataType() 
== DataType.MATRIX
+ 

[03/23] systemml git commit: Add name to sorting of EMult rewrite. Handle Ternary A*A*B case.

[mboehm7]

Add name to sorting of EMult rewrite. Handle Ternary A*A*B case.

AggUnaryOp now constructs the TernaryOperator (A,A,B) instead of (A^2,B,1).


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/ff8c836c
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/ff8c836c
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/ff8c836c

Branch: refs/heads/master
Commit: ff8c836c7b736dbd7b7651ac792a6d8c23989c98
Parents: eb0599d
Author: Dylan Hutchison 
Authored: Fri Jun 9 13:18:19 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:18 2017 -0700

--
 .../java/org/apache/sysml/hops/AggUnaryOp.java  | 62 ++--
 .../apache/sysml/hops/rewrite/RewriteEMult.java |  4 +-
 2 files changed, 48 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/ff8c836c/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
--
diff --git a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java 
b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
index ee4ded2..4573b66 100644
--- a/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
+++ b/src/main/java/org/apache/sysml/hops/AggUnaryOp.java
@@ -516,7 +516,6 @@ public class AggUnaryOp extends Hop implements 
MultiThreadedHop
}
}
}
-   
return ret;
}

@@ -631,24 +630,53 @@ public class AggUnaryOp extends Hop implements 
MultiThreadedHop
Hop input11 = input1.getInput().get(0);
Hop input12 = input1.getInput().get(1);

-   Lop in1 = null;
-   Lop in2 = null;
-   Lop in3 = null;
+   Lop in1 = null, in2 = null, in3 = null;
+   boolean handled = false;

-   if( input11 instanceof BinaryOp && 
((BinaryOp)input11).getOp()==OpOp2.MULT )
-   {
-   in1 = input11.getInput().get(0).constructLops();
-   in2 = input11.getInput().get(1).constructLops();
-   in3 = input12.constructLops();
-   }
-   else if( input12 instanceof BinaryOp && 
((BinaryOp)input12).getOp()==OpOp2.MULT )
-   {
-   in1 = input11.constructLops();
-   in2 = input12.getInput().get(0).constructLops();
-   in3 = input12.getInput().get(1).constructLops();
+   if( input11 instanceof BinaryOp ) {
+   BinaryOp b11 = (BinaryOp)input11;
+   switch (b11.getOp()) {
+   case MULT: // A*B*C case
+   in1 = input11.getInput().get(0).constructLops();
+   in2 = input11.getInput().get(1).constructLops();
+   in3 = input12.constructLops();
+   handled = true;
+   break;
+   case POW: // A*A*B case
+   Hop b112 = b11.getInput().get(1);
+   if ( !(input12 instanceof BinaryOp && 
((BinaryOp)input12).getOp()==OpOp2.MULT)
+   && b112 instanceof LiteralOp
+   && 
((LiteralOp)b112).getLongValue() == 2) {
+   in1 = 
b11.getInput().get(0).constructLops();
+   in2 = in1;
+   in3 = input12.constructLops();
+   handled = true;
+   }
+   break;
+   }
+   } else if( input12 instanceof BinaryOp ) {
+   BinaryOp b12 = (BinaryOp)input12;
+   switch (b12.getOp()) {
+   case MULT: // A*B*C case
+   in1 = input11.constructLops();
+   in2 = input12.getInput().get(0).constructLops();
+   in3 = input12.getInput().get(1).constructLops();
+   handled = true;
+   break;
+   case POW: // A*B*B case
+   Hop b112 = b12.getInput().get(1);
+   if ( b112 instanceof LiteralOp
+   && 
((LiteralOp)b112).getLongValue() == 2) {
+   in1 = 
b12.getInput().get(0).constructLops();
+   in2 = in1;
+ 

[13/23] systemml git commit: Relax tolerance for ElementwiseAdditionMultiplicationTest

[mboehm7]

Relax tolerance for ElementwiseAdditionMultiplicationTest

The new RewriteElementwiseMultChainOptimization reorders
`(A*B)*C` to `A*(B*C)`, which causes the result not to be exact.
 Use epsilon of 1e-10.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/de469d23
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/de469d23
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/de469d23

Branch: refs/heads/master
Commit: de469d235e5fe06fd3e13a32262d6c357ffdcc81
Parents: 0a8936c
Author: Dylan Hutchison 
Authored: Sun Jun 11 12:50:39 2017 -0700
Committer: Dylan Hutchison 
Committed: Sun Jun 18 17:43:51 2017 -0700

--
 .../binary/matrix/ElementwiseAdditionMultiplicationTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/de469d23/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionMultiplicationTest.java
--
diff --git 
a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionMultiplicationTest.java
 
b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionMultiplicationTest.java
index 523a648..f78e598 100644
--- 
a/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionMultiplicationTest.java
+++ 
b/src/test/java/org/apache/sysml/test/integration/functions/binary/matrix/ElementwiseAdditionMultiplicationTest.java
@@ -134,6 +134,6 @@ public class ElementwiseAdditionMultiplicationTest extends 
AutomatedTestBase
 
runTest();
 
-   compareResults();
+   compareResults(1e-10);
}
 }

[1/2] systemml git commit: [SYSTEMML-1773] JMLC error handling invalid argument/variable names

[mboehm7]

Repository: systemml
Updated Branches:
  refs/heads/master 6778a63b0 -> 1b3dff06b


[SYSTEMML-1773] JMLC error handling invalid argument/variable names

This patch improves the error handling of invalid argument and variable
names passed to JMLC prepared scripts. Input/variables refer to the
left-hand-side of read assignments, while arguments (aka $ parameters)
can only be used on the right hand-side (potentially used in ifdefs). We
now explicitly check that variables do not start with $, while arguments
have to start with $. This helps to avoid silent errors, where passed
arguments with invalid names are simply ignored.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/4ca4d34f
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/4ca4d34f
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/4ca4d34f

Branch: refs/heads/master
Commit: 4ca4d34fe77b6c0726a2c7b06fbe069e956cdc25
Parents: 6778a63
Author: Matthias Boehm 
Authored: Fri Jul 14 19:20:42 2017 -0700
Committer: Matthias Boehm 
Committed: Fri Jul 14 20:49:22 2017 -0700

--
 .../org/apache/sysml/api/jmlc/Connection.java   | 27 +---
 1 file changed, 24 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/4ca4d34f/src/main/java/org/apache/sysml/api/jmlc/Connection.java
--
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java 
b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index 7739a35..be440c8 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -25,9 +25,13 @@ import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.sysml.api.DMLException;
@@ -43,6 +47,7 @@ import 
org.apache.sysml.hops.rewrite.RewriteRemovePersistentReadWrite;
 import org.apache.sysml.parser.DMLProgram;
 import org.apache.sysml.parser.DMLTranslator;
 import org.apache.sysml.parser.DataExpression;
+import org.apache.sysml.parser.LanguageException;
 import org.apache.sysml.parser.ParseException;
 import org.apache.sysml.parser.ParserFactory;
 import org.apache.sysml.parser.ParserWrapper;
@@ -176,12 +181,21 @@ public class Connection implements Closeable
{
DMLScript.SCRIPT_TYPE = parsePyDML ? ScriptType.PYDML : 
ScriptType.DML;
 
-   //prepare arguments
+   //check for valid names of passed arguments
+   String[] invalidArgs = args.keySet().stream()
+   .filter(k -> k==null || 
!k.startsWith("$")).toArray(String[]::new);
+   if( invalidArgs.length > 0 )
+   throw new LanguageException("Invalid argument names: 
"+Arrays.toString(invalidArgs));
+   
+   //check for valid names of input and output variables
+   String[] invalidVars = asSet(inputs, outputs).stream()
+   .filter(k -> k==null || 
k.startsWith("$")).toArray(String[]::new);
+   if( invalidVars.length > 0 )
+   throw new LanguageException("Invalid variable names: 
"+Arrays.toString(invalidVars));

//simplified compilation chain
Program rtprog = null;
-   try
-   {
+   try {
//parsing
ParserWrapper parser = 
ParserFactory.createParser(parsePyDML ? ScriptType.PYDML : ScriptType.DML);
DMLProgram prog = parser.parse(null, script, args);
@@ -828,4 +842,11 @@ public class Connection implements Closeable
public FrameBlock readTransformMetaDataFromPath(String spec, String 
metapath, String colDelim) throws IOException {
return TfMetaUtils.readTransformMetaDataFromPath(spec, 
metapath, colDelim);
}
+   
+   private Set asSet(String[] inputs, String[] outputs) {
+   Set ret = new HashSet();
+   CollectionUtils.addAll(ret, inputs);
+   CollectionUtils.addAll(ret, outputs);
+   return ret;
+   }
 }

systemml git commit: [SYSTEMML-1419] Cleanup nested if-elses in GLM and GLM-predict

[deron]

Repository: systemml
Updated Branches:
  refs/heads/master a4ce06461 -> ccac6dd37


[SYSTEMML-1419] Cleanup nested if-elses in GLM and GLM-predict

Closes #562.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/ccac6dd3
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/ccac6dd3
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/ccac6dd3

Branch: refs/heads/master
Commit: ccac6dd37de3dba745069caeab9803aa5d69190f
Parents: a4ce064
Author: Janardhan 
Authored: Fri Jul 14 10:29:55 2017 -0700
Committer: Deron Eriksson 
Committed: Fri Jul 14 10:29:55 2017 -0700

--
 scripts/algorithms/GLM-predict.dml |  40 +++
 scripts/algorithms/GLM.dml | 202 
 2 files changed, 123 insertions(+), 119 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/ccac6dd3/scripts/algorithms/GLM-predict.dml
--
diff --git a/scripts/algorithms/GLM-predict.dml 
b/scripts/algorithms/GLM-predict.dml
index 251d85a..49a593c 100644
--- a/scripts/algorithms/GLM-predict.dml
+++ b/scripts/algorithms/GLM-predict.dml
@@ -105,13 +105,13 @@ dispersion = as.double (dispersion);
 
 if (dist_type == 3) {
 link_type = 2;
-} else { if (link_type == 0) { # Canonical Link
+} else if (link_type == 0) { # Canonical Link
 if (dist_type == 1) {
 link_type = 1;
 link_power = 1.0 - var_power;
-} else { if (dist_type == 2) {
+} else if (dist_type == 2) {
 link_type = 2;
-}}} }
+}   }
 
 X = read (fileX);
 num_records  = nrow (X);
@@ -343,36 +343,36 @@ glm_means_and_vars =
 # POWER DISTRIBUTION
 if  (link_power ==  0) {
 y_mean = exp (linear_terms);
-} else { if (link_power ==  1.0) {
+} else if (link_power ==  1.0) {
 y_mean = linear_terms;
-} else { if (link_power == -1.0) {
+} else if (link_power == -1.0) {
 y_mean = 1.0 / linear_terms;
 } else {
 y_mean = linear_terms ^ (1.0 / link_power);
-}}}
+}
 if (var_power == 0) {
 var_function = matrix (1.0, rows = num_points, cols = 1);
-} else { if (var_power == 1.0) {
+} else if (var_power == 1.0) {
 var_function = y_mean;
 } else {
 var_function = y_mean ^ var_power;
-}}
+}
 means = y_mean;
 vars = var_function;
-} else { if (dist_type == 2 & link_type >= 1 & link_type <= 5) {
+} else if (dist_type == 2 & link_type >= 1 & link_type <= 5) {
 # BINOMIAL/BERNOULLI DISTRIBUTION
 y_prob = matrix (0.0, rows = num_points, cols = 2);
 if  (link_type == 1 & link_power == 0)  { # Binomial.log
 y_prob [, 1]  = exp (linear_terms);
 y_prob [, 2]  = 1.0 - y_prob [, 1];
-} else { if (link_type == 1 & link_power != 0)  { # 
Binomial.power_nonlog
+} else if (link_type == 1 & link_power != 0)  { # Binomial.power_nonlog
 y_prob [, 1]  = linear_terms ^ (1.0 / link_power);
 y_prob [, 2]  = 1.0 - y_prob [, 1];
-} else { if (link_type == 2)  { # Binomial.logit
+} else if (link_type == 2)  { # Binomial.logit
 elt = exp (linear_terms);
 y_prob [, 1]  = elt / (1.0 + elt);
 y_prob [, 2]  = 1.0 / (1.0 + elt);
-} else { if (link_type == 3)  { # Binomial.probit
+} else if (link_type == 3)  { # Binomial.probit
 sign_lt = 2 * (linear_terms >= 0) - 1;
 t_gp = 1.0 / (1.0 + abs (linear_terms) * 0.231641888);  # 
0.231641888 = 0.3275911 / sqrt (2.0)
 erf_corr =
@@ -384,20 +384,20 @@ glm_means_and_vars =
 y_prob [, 1] = (1 + sign_lt) - erf_corr;
 y_prob [, 2] = (1 - sign_lt) + erf_corr;
 y_prob = y_prob / 2;
-} else { if (link_type == 4)  { # Binomial.cloglog
+} else if (link_type == 4)  { # Binomial.cloglog
 elt = exp (linear_terms);
 is_too_small = ((1000 + elt) == 1000);
 y_prob [, 2] = exp (- elt);
 y_prob [, 1] = (1 - is_too_small) * (1.0 - y_prob [, 2]) + 
is_too_small * elt * (1.0 - elt / 2);
-} else { if (link_type == 5)  { # Binomial.cauchit
+} else if (link_type == 5)  { # Binomial.cauchit
 atan_linear_terms = atan (linear_terms);
 y_prob [, 1] = 0.5 + atan_linear_terms / 
3.1415926535897932384626433832795;
 y_prob [, 2] = 0.5 - atan_linear_terms / 

[2/2] systemml git commit: [SYSTEMML-1768] Cleanup properties of systemml-config file

[mboehm7]

[SYSTEMML-1768] Cleanup properties of systemml-config file

This patch cleans up the following two properties of the
SystemML-config.xml file in order to better convey their meaning:

1) cp.parallel.matrixmult -> cp.parallel.ops
2) cp.parallel.textio -> cp.parallel.io


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/a4ce0646
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/a4ce0646
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/a4ce0646

Branch: refs/heads/master
Commit: a4ce06461deedd9c4f9d0293195ce81ae42ccfd6
Parents: 586c67b
Author: Matthias Boehm 
Authored: Thu Jul 13 19:46:08 2017 -0700
Committer: Matthias Boehm 
Committed: Thu Jul 13 19:46:26 2017 -0700

--
 conf/SystemML-config.xml.template   |  8 +++
 docs/standalone-guide.md|  4 ++--
 .../java/org/apache/sysml/conf/DMLConfig.java   | 25 ++--
 .../org/apache/sysml/hops/OptimizerUtils.java   |  4 ++--
 src/main/standalone/SystemML-config.xml |  8 +++
 src/test/config/SystemML-config.xml |  8 +++
 .../functions/mlcontext/GNMFTest.java   |  4 ++--
 .../SystemML-config-codegen-compress.xml|  8 +++
 .../codegen/SystemML-config-codegen.xml |  8 +++
 .../codegen/SystemML-config-codegen6.xml|  8 +++
 .../compress/SystemML-config-compress.xml   |  8 +++
 .../functions/dmlscript/SystemML-config.xml |  4 ++--
 .../gdfo/SystemML-config-globalopt.xml  |  8 +++
 13 files changed, 52 insertions(+), 53 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/a4ce0646/conf/SystemML-config.xml.template
--
diff --git a/conf/SystemML-config.xml.template 
b/conf/SystemML-config.xml.template
index e026f8e..ff724b4 100644
--- a/conf/SystemML-config.xml.template
+++ b/conf/SystemML-config.xml.template
@@ -48,11 +48,11 @@

default

-   
-   true
+   
+   true

-   
-   true
+   
+   true


false

http://git-wip-us.apache.org/repos/asf/systemml/blob/a4ce0646/docs/standalone-guide.md
--
diff --git a/docs/standalone-guide.md b/docs/standalone-guide.md
index 4f901c1..a401c30 100644
--- a/docs/standalone-guide.md
+++ b/docs/standalone-guide.md
@@ -334,8 +334,8 @@ The console output should show the accuracy of the trained 
model in percent, i.e
 15/09/01 01:32:51 INFO conf.DMLConfig: Updating dml.yarn.appmaster.mem 
with value 2048
 15/09/01 01:32:51 INFO conf.DMLConfig: Updating dml.yarn.mapreduce.mem 
with value 2048
 15/09/01 01:32:51 INFO conf.DMLConfig: Updating dml.yarn.app.queue with 
value default
-15/09/01 01:32:51 INFO conf.DMLConfig: Updating cp.parallel.matrixmult 
with value true
-15/09/01 01:32:51 INFO conf.DMLConfig: Updating cp.parallel.textio with 
value true
+15/09/01 01:32:51 INFO conf.DMLConfig: Updating cp.parallel.ops with value 
true
+15/09/01 01:32:51 INFO conf.DMLConfig: Updating cp.parallel.io with value 
true
 Accuracy (%): 74.14965986394557
 15/09/01 01:32:52 INFO api.DMLScript: SystemML Statistics:
 Total execution time:  0.130 sec.

http://git-wip-us.apache.org/repos/asf/systemml/blob/a4ce0646/src/main/java/org/apache/sysml/conf/DMLConfig.java
--
diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java 
b/src/main/java/org/apache/sysml/conf/DMLConfig.java
index 00a591c..5b5b8ea 100644
--- a/src/main/java/org/apache/sysml/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java
@@ -68,15 +68,15 @@ public class DMLConfig
public static final String YARN_APPMASTERMEM= 
"dml.yarn.appmaster.mem"; 
public static final String YARN_MAPREDUCEMEM= 
"dml.yarn.mapreduce.mem"; 
public static final String YARN_APPQUEUE= "dml.yarn.app.queue"; 
-   public static final String CP_PARALLEL_MATRIXMULT = 
"cp.parallel.matrixmult";
-   public static final String CP_PARALLEL_TEXTIO   = "cp.parallel.textio";
+   public static final String CP_PARALLEL_OPS  = "cp.parallel.ops";
+   public static final String CP_PARALLEL_IO   = "cp.parallel.io";
public static final String COMPRESSED_LINALG= "compressed.linalg";
-   public static final String NATIVE_BLAS  = 
"native.blas";
+   public static final String NATIVE_BLAS  = "native.blas";
public static final String CODEGEN  = "codegen.enabled"; 
//boolean
public static final String CODEGEN_PLANCACHE= "codegen.plancache"; 
//boolean
public 

[1/2] systemml git commit: [SYSTEMML-1765] Support for reading dml scripts from object stores

[mboehm7]

Repository: systemml
Updated Branches:
  refs/heads/master f31548007 -> a4ce06461


[SYSTEMML-1765] Support for reading dml scripts from object stores

This patch generates the various methods for reading dml scripts files
to support (apart from local fs and hdfs) also the read from object
stores such as swift and s3.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/586c67b6
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/586c67b6
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/586c67b6

Branch: refs/heads/master
Commit: 586c67b6a47950305f1bb57ba809aa295a83861b
Parents: f315480
Author: Matthias Boehm 
Authored: Thu Jul 13 17:20:20 2017 -0700
Committer: Matthias Boehm 
Committed: Thu Jul 13 19:46:25 2017 -0700

--
 src/main/java/org/apache/sysml/api/DMLScript.java |  7 +++
 .../java/org/apache/sysml/api/jmlc/Connection.java|  4 ++--
 .../org/apache/sysml/api/mlcontext/ScriptFactory.java | 14 +++---
 src/main/java/org/apache/sysml/conf/DMLConfig.java|  4 ++--
 .../java/org/apache/sysml/parser/ParserWrapper.java   |  7 ---
 5 files changed, 18 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/586c67b6/src/main/java/org/apache/sysml/api/DMLScript.java
--
diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java 
b/src/main/java/org/apache/sysml/api/DMLScript.java
index 6dd7e89..515d632 100644
--- a/src/main/java/org/apache/sysml/api/DMLScript.java
+++ b/src/main/java/org/apache/sysml/api/DMLScript.java
@@ -567,8 +567,8 @@ public class DMLScript
try 
{
//read from hdfs or gpfs file system
-   if(fileName.startsWith("hdfs:")
-   || fileName.startsWith("gpfs:") )
+   if(fileName.startsWith("hdfs:") || 
fileName.startsWith("gpfs:")
+   || 
IOUtilFunctions.isObjectStoreFileScheme(new Path(fileName)) )
{ 
Path scriptPath = new Path(fileName);
FileSystem fs = 
IOUtilFunctions.getFileSystem(scriptPath);
@@ -588,8 +588,7 @@ public class DMLScript
sb.append( "\n" );
}
}
-   catch (IOException ex)
-   {
+   catch (IOException ex) {
LOG.error("Failed to read the script from the 
file system", ex);
throw ex;
}

http://git-wip-us.apache.org/repos/asf/systemml/blob/586c67b6/src/main/java/org/apache/sysml/api/jmlc/Connection.java
--
diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java 
b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
index af50139..7739a35 100644
--- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java
+++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java
@@ -250,8 +250,8 @@ public class Connection implements Closeable
try 
{
//read from hdfs or gpfs file system
-   if(fname.startsWith("hdfs:") 
-   || fname.startsWith("gpfs:") ) 
+   if(fname.startsWith("hdfs:") || 
fname.startsWith("gpfs:")
+   || IOUtilFunctions.isObjectStoreFileScheme(new 
Path(fname)) ) 
{ 
Path scriptPath = new Path(fname);
FileSystem fs = 
IOUtilFunctions.getFileSystem(scriptPath);

http://git-wip-us.apache.org/repos/asf/systemml/blob/586c67b6/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
--
diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java 
b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
index 4a30f00..10cd947 100644
--- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
+++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java
@@ -351,20 +351,20 @@ public class ScriptFactory {
throw new MLContextException("Script file path is 
null");
}
try {
-   if (scriptFilePath.startsWith("hdfs:") || 
scriptFilePath.startsWith("gpfs:")) {
+   if (