[4/4] incubator-systemml git commit: [SYSTEMML-1547] Cache-conscious tsmm post-processing (copy upper, nnz)

2017-04-19 Thread mboehm7 
[SYSTEMML-1547] Cache-conscious tsmm post-processing (copy upper, nnz)

This patch improves the performance of tsmm postprocessing including (1)
the copy of the upper triangular matrix to the lower triangle, and (2)
the number of non-zeros computation of the output. We now use a
cache-conscious scheme of copying similar to dense-dense transpose
including the fused maintenance of nnz. 

On a scenario of 10 iterations running outer products of 15K vectors
(with 15K x 15K outputs), this patch improved end-to-end execution time
from 16.2s to 11.4s 

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/117ea480
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/117ea480
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/117ea480

Branch: refs/heads/master
Commit: 117ea480dadbdaf8be4318eddbdf23018ce50544
Parents: b641edc
Author: Matthias Boehm 
Authored: Wed Apr 19 18:39:26 2017 -0700
Committer: Matthias Boehm 
Committed: Wed Apr 19 18:46:31 2017 -0700

--
 .../runtime/matrix/data/LibMatrixMult.java  | 60 ++--
 .../runtime/matrix/data/LibMatrixReorg.java |  2 +-
 2 files changed, 45 insertions(+), 17 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/117ea480/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
--
diff --git 
a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java 
b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
index d746aa3..5c7c2ef 100644
--- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
+++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java
@@ -385,8 +385,8 @@ public class LibMatrixMult
matrixMultTransposeSelfDense(m1, ret, leftTranspose, 0, 
ret.rlen );
 
//post-processing
-   copyUpperToLowerTriangle( ret );
-   ret.recomputeNonZeros();
+   long nnz = copyUpperToLowerTriangle(ret);
+   ret.setNonZeros(nnz);
ret.examSparsity(); 

//System.out.println("TSMM 
("+m1.isInSparseFormat()+","+m1.getNumRows()+","+m1.getNumColumns()+","+m1.getNonZeros()+","+leftTranspose+")
 in "+time.stop());
@@ -436,8 +436,8 @@ public class LibMatrixMult
}

//post-processing
-   copyUpperToLowerTriangle( ret );
-   ret.recomputeNonZeros();
+   long nnz = copyUpperToLowerTriangle(ret);   
+   ret.setNonZeros(nnz);
ret.examSparsity(); 

//System.out.println("TSMM k="+k+" 
("+m1.isInSparseFormat()+","+m1.getNumRows()+","+m1.getNumColumns()+","+m1.getNonZeros()+","+leftTranspose+")
 in "+time.stop());
@@ -3413,17 +3413,47 @@ public class LibMatrixMult
 * result down to lower triangular matrix once.
 * 
 * @param ret matrix
+* @return number of non zeros
 */
-   public static void copyUpperToLowerTriangle( MatrixBlock ret )
-   {
-   double[] c = ret.denseBlock;
-   final int m = ret.rlen;
-   final int n = ret.clen;
+   public static long copyUpperToLowerTriangle( MatrixBlock ret )
+   {
+   //ret is guaranteed to be a squared, symmetric matrix
+   if( ret.rlen != ret.clen )
+   throw new RuntimeException("Invalid non-squared input 
matrix.");
+   
+   final double[] c = ret.denseBlock;
+   final int n = ret.rlen;
+   long nnz = 0;
+   
+   //blocked execution (2x128KB for L2 blocking)
+   final int blocksizeIJ = 128; 
+   
+   //handle blocks on diagonal
+   for( int bi = 0; bi

[1/4] incubator-systemml git commit: [MINOR] Graceful value type casting of scalar function args, cleanup

2017-04-19 Thread mboehm7 
Repository: incubator-systemml
Updated Branches:
  refs/heads/master d69fdfe45 -> 117ea480d


[MINOR] Graceful value type casting of scalar function args, cleanup 

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/53fe1ae6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/53fe1ae6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/53fe1ae6

Branch: refs/heads/master
Commit: 53fe1ae68ab3b5024ead0d258a213f3e4f392616
Parents: d69fdfe
Author: Matthias Boehm 
Authored: Tue Apr 18 01:37:55 2017 -0700
Committer: Matthias Boehm 
Committed: Wed Apr 19 18:46:28 2017 -0700

--
 .../sysml/debug/DMLDebuggerFunctions.java   | 31 ++---
 .../sysml/hops/ipa/InterProceduralAnalysis.java | 17 ++---
 .../sysml/hops/rewrite/HopRewriteUtils.java | 27 +++
 .../context/ExecutionContext.java   | 29 ++--
 .../cp/FunctionCallCPInstruction.java   | 11 +-
 .../instructions/cp/ScalarObjectFactory.java| 36 ++--
 6 files changed, 58 insertions(+), 93 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/53fe1ae6/src/main/java/org/apache/sysml/debug/DMLDebuggerFunctions.java
--
diff --git a/src/main/java/org/apache/sysml/debug/DMLDebuggerFunctions.java 
b/src/main/java/org/apache/sysml/debug/DMLDebuggerFunctions.java
index d852747..09bb0d6 100644
--- a/src/main/java/org/apache/sysml/debug/DMLDebuggerFunctions.java
+++ b/src/main/java/org/apache/sysml/debug/DMLDebuggerFunctions.java
@@ -34,14 +34,9 @@ import 
org.apache.sysml.runtime.controlprogram.LocalVariableMap;
 import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
 import org.apache.sysml.runtime.instructions.Instruction;
 import org.apache.sysml.runtime.instructions.MRJobInstruction;
-import org.apache.sysml.runtime.instructions.cp.BooleanObject;
 import org.apache.sysml.runtime.instructions.cp.BreakPointInstruction;
 import org.apache.sysml.runtime.instructions.cp.CPInstruction;
-import org.apache.sysml.runtime.instructions.cp.Data;
-import org.apache.sysml.runtime.instructions.cp.DoubleObject;
-import org.apache.sysml.runtime.instructions.cp.IntObject;
-import org.apache.sysml.runtime.instructions.cp.ScalarObject;
-import org.apache.sysml.runtime.instructions.cp.StringObject;
+import org.apache.sysml.runtime.instructions.cp.ScalarObjectFactory;
 import 
org.apache.sysml.runtime.instructions.cp.BreakPointInstruction.BPINSTRUCTION_STATUS;
 import org.apache.sysml.runtime.matrix.data.MatrixBlock;
 
@@ -269,28 +264,8 @@ public class DMLDebuggerFunctions {
if (variables != null && !variables.keySet().isEmpty()) {
if (variables.get(varname) != null) {
if (variables.get(varname).getDataType() == 
DataType.SCALAR) {
-   Data value;
-   
switch(variables.get(varname).getValueType()) {
-   case DOUBLE:
-   double d = 
Double.parseDouble(args[1]);
-   value = (ScalarObject) 
new DoubleObject(d);
-   break;
-   case INT:
-   long i = 
Long.parseLong(args[1]);
-   value = (ScalarObject) 
new IntObject(i);
-   break;
-   case BOOLEAN:
-   boolean b = 
Boolean.parseBoolean(args[1]);
-   value = (ScalarObject) 
new BooleanObject(b);
-   break;
-   case STRING:
-   value = (ScalarObject) 
new StringObject(args[1]);
-   break;
-   default:
-   
System.err.println("Invalid scalar value type.");
-   return;
-   }
-   variables.put(varname, value);
+   variables.put(varname, 
ScalarObjectFactory
+

[3/4] incubator-systemml git commit: [SYSTEMML-1321] Improved dynamic recompilation (stats update p rewrites)

2017-04-19 Thread mboehm7 
[SYSTEMML-1321] Improved dynamic recompilation (stats update p rewrites)

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/b641edc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/b641edc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/b641edc1

Branch: refs/heads/master
Commit: b641edc165aab129559a5db1bf5e9d050cd30b37
Parents: 7b7b9ba
Author: Matthias Boehm 
Authored: Tue Apr 18 14:27:39 2017 -0700
Committer: Matthias Boehm 
Committed: Wed Apr 19 18:46:30 2017 -0700

--
 .../org/apache/sysml/hops/recompile/Recompiler.java  | 15 +--
 1 file changed, 13 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/b641edc1/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
--
diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java 
b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
index 6b74bf7..6c99d16 100644
--- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
+++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java
@@ -198,8 +198,14 @@ public class Recompiler
rUpdateStatistics( hopRoot, vars );

// dynamic hop rewrites
-   if( !inplace )
+   if( !inplace ) {
_rewriter.get().rewriteHopDAGs( hops, null );
+   
+   //update stats after rewrites
+   Hop.resetVisitStatus(hops);
+   for( Hop hopRoot : hops )
+   rUpdateStatistics( hopRoot, vars );
+   }

// refresh memory estimates (based on updated stats,
// before: init memo table with propagated worst-case 
estimates,
@@ -303,8 +309,13 @@ public class Recompiler
rUpdateStatistics( hops, vars );

// dynamic hop rewrites
-   if( !inplace )
+   if( !inplace ) {
_rewriter.get().rewriteHopDAG( hops, null );
+   
+   //update stats after rewrites
+   hops.resetVisitStatus();
+   rUpdateStatistics( hops, vars );
+   }

// refresh memory estimates (based on updated stats)
MemoTable memo = new MemoTable();

[incubator-systemml] Git Push Summary

2017-04-19 Thread acs_s 
Repository: incubator-systemml
Updated Branches:
  refs/heads/branch-0.14 [created] 8bdcf106c

svn commit: r1792001 - /incubator/systemml/site/get-started.html

2017-04-19 Thread deron 
Author: deron
Date: Wed Apr 19 23:05:19 2017
New Revision: 1792001

URL: http://svn.apache.org/viewvc?rev=1792001=rev
Log:
Fix 0.12 download link on get-started

Modified:
incubator/systemml/site/get-started.html

Modified: incubator/systemml/site/get-started.html
URL: 
http://svn.apache.org/viewvc/incubator/systemml/site/get-started.html?rev=1792001=1792000=1792001=diff
==
--- incubator/systemml/site/get-started.html (original)
+++ incubator/systemml/site/get-started.html Wed Apr 19 23:05:19 2017
@@ -247,7 +247,7 @@ pip3 install systemml
 
 

-   Alternatively, if you intend to use SystemML via spark-shell (or 
spark-submit), you only need systemml-0.12.0-incubating.jar, which is packaged 
into our official binary release (http://www.apache.org/dyn/closer.cgi/pub/apache/incubator/systemml/0.12.0-incubating/systemml-0.12.0-incubating.zip;
 target="_blank">systemml-0.12.0-incubating.zip).
+   Alternatively, if you intend to use SystemML via spark-shell (or 
spark-submit), you only need systemml-0.12.0-incubating.jar, which is packaged 
into our official binary release (http://www.apache.org/dyn/closer.lua/incubator/systemml/0.12.0-incubating/systemml-0.12.0-incubating-bin.zip;
 target="_blank">systemml-0.12.0-incubating.zip).
Note: If you have installed SystemML via pip, you can get the location 
of this jar by executing following command:

   python -c 'import imp; import os; print 
os.path.join(imp.find_module("systemml")[1], 
"systemml-java")'

[1/2] incubator-systemml git commit: [SYSTEMML-692] Added initial version of DML generator for Caffe

2017-04-19 Thread niketanpansare 
Repository: incubator-systemml
Updated Branches:
  refs/heads/gh-pages b91f9bfec -> c4918f5b6


[SYSTEMML-692] Added initial version of DML generator for Caffe

This experimental interface is called Caffe2DML and doesnot affect other 
functionality.

- Updated the interface to match the Caffe specification as per
  @bertholdreinwald 's suggestion.
- Added support for fine-tuning.
- Added support for explain, statistics and gpu.

Closes #422.


Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/48ba441e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/48ba441e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/48ba441e

Branch: refs/heads/gh-pages
Commit: 48ba441ee3d3ad440322c72c6a3f8a55dfcd84bc
Parents: b91f9bf
Author: Niketan Pansare 
Authored: Wed Apr 19 14:07:44 2017 -0800
Committer: Niketan Pansare 
Committed: Wed Apr 19 15:07:43 2017 -0700

--
 beginners-guide-caffe2dml.md | 124 ++
 devdocs/deep-learning.md |  84 ++
 2 files changed, 208 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/48ba441e/beginners-guide-caffe2dml.md
--
diff --git a/beginners-guide-caffe2dml.md b/beginners-guide-caffe2dml.md
new file mode 100644
index 000..cfcc0cb
--- /dev/null
+++ b/beginners-guide-caffe2dml.md
@@ -0,0 +1,124 @@
+---
+layout: global
+title: Beginner's Guide for Caffe2DML users
+description: Beginner's Guide for Caffe2DML users
+---
+
+
+* This will become a table of contents (this text will be scraped).
+{:toc}
+
+
+
+## Introduction
+
+Caffe2DML is an experimental API that converts an Caffe specification to DML.
+
+## Frequently asked questions
+
+- How to set batch size ?
+
+Batch size is set in `data_param` of the Data layer:
+
+   layer {
+ name: "mnist"
+ type: "Data"
+ top: "data"
+ top: "label"
+ data_param {
+   source: "mnist_train"
+   batch_size: 64
+   backend: LMDB
+ }
+   }
+   
+- How to set maximum number of iterations for training ?
+
+Caffe allows you to set the maximum number of iterations in solver 
specification
+
+   # The maximum number of iterations
+   max_iter: 2000
+   
+- How to set the size of the validation dataset ?
+
+The size of the validation dataset is determined by the parameters `test_iter` 
and the batch size. For example: If the batch size is 64 and 
+`test_iter` is 10, then the validation size is 640. This setting generates 
following DML code internally:
+
+   num_images = nrow(y_full)
+   BATCH_SIZE = 64
+   num_validation = 10 * BATCH_SIZE
+   X = X_full[(num_validation+1):num_images,]; y = 
y_full[(num_validation+1):num_images,]
+   X_val = X_full[1:num_validation,]; y_val = y_full[1:num_validation,]
+   num_images = nrow(y) 
+
+- How to monitor loss via command-line ?
+
+To monitor loss, please set following parameters in the solver specification
+
+   # Display training loss and accuracy every 100 iterations
+   display: 100
+   # Carry out validation every 500 training iterations and display 
validation loss and accuracy.
+   test_iter: 10
+   test_interval: 500
+   
+ - How to pass a single jpeg image to Caffe2DML for prediction ?
+ 
+   from PIL import Image
+   import systemml as sml
+   from systemml.mllearn import Caffe2DML
+   img_shape = (3, 224, 224)
+   input_image = sml.convertImageToNumPyArr(Image.open(img_file_path), 
img_shape=img_shape)
+   resnet = Caffe2DML(sqlCtx, solver='ResNet_50_solver.proto', 
weights='ResNet_50_pretrained_weights', input_shape=img_shape)
+   resnet.predict(input_image)
+
+- How to prepare a directory of jpeg images for training with Caffe2DML ?
+
+The below example assumes that the input dataset has 2 labels `cat` and `dogs` 
and the filename has these labels as prefix.
+We iterate through the directory and convert each jpeg image into 
pyspark.ml.linalg.Vector using pyspark.
+These vectors are stored as DataFrame and randomized using Spark SQL's 
`orderBy(rand())` function.
+The DataFrame is then saved in parquet format to reduce the cost of 
preprocessing for repeated training.
+
+   from systemml.mllearn import Caffe2DML
+   from pyspark.sql import SQLContext
+   import numpy as np
+   import urllib, os, scipy.ndimage
+   from pyspark.ml.linalg import Vectors
+   from pyspark import StorageLevel
+   import systemml as sml
+   from pyspark.sql.functions import rand 
+   # ImageNet specific parameters
+   img_shape = (3, 224, 224)
+   train_dir

[2/2] incubator-systemml git commit: [MINOR] Updated the algorithm documentation to clearly mark the "Argument" section.

2017-04-19 Thread niketanpansare 
[MINOR] Updated the algorithm documentation to clearly mark the "Argument" 
section.


Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/c4918f5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/c4918f5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/c4918f5b

Branch: refs/heads/gh-pages
Commit: c4918f5b64befaebfb0cbd47c02c50cb84642a12
Parents: 48ba441
Author: Niketan Pansare 
Authored: Wed Apr 19 14:20:00 2017 -0800
Committer: Niketan Pansare 
Committed: Wed Apr 19 15:21:39 2017 -0700

--
 algorithms-classification.md | 12 ++--
 algorithms-regression.md | 10 +-
 2 files changed, 11 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c4918f5b/algorithms-classification.md
--
diff --git a/algorithms-classification.md b/algorithms-classification.md
index 0ee43bf..b029e0a 100644
--- a/algorithms-classification.md
+++ b/algorithms-classification.md
@@ -180,7 +180,7 @@ val prediction = model.transform(X_test_df)
 
 
 
-### Arguments
+### Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) to read the input matrix of feature vectors; each row
 constitutes one feature vector.
@@ -592,7 +592,7 @@ val prediction = model.transform(X_test_df)
 
 
 
- Arguments
+ Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) to read the matrix of feature vectors; each
 row constitutes one feature vector.
@@ -861,7 +861,7 @@ val prediction = model.transform(X_test_df)
 
 
 
- Arguments
+ Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) containing the explanatory variables in
 a matrix. Each row constitutes an example.
@@ -1212,7 +1212,7 @@ val prediction = model.transform(X_test_df)
 
 
 
-### Arguments
+### Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) to read the matrix of feature vectors; each
 row constitutes one feature vector.
@@ -1472,7 +1472,7 @@ implementation is well-suited to handle large-scale data 
and builds a
 
 
 
-### Arguments
+### Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) to read the matrix of feature vectors; each row
 constitutes one feature vector. Note that categorical features in $X$
@@ -1887,7 +1887,7 @@ for classification in parallel.
 
 
 
-### Arguments
+### Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) to read the matrix of feature vectors; each row
 constitutes one feature vector. Note that categorical features in $X$

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/c4918f5b/algorithms-regression.md
--
diff --git a/algorithms-regression.md b/algorithms-regression.md
index 80b38a3..31f7ec2 100644
--- a/algorithms-regression.md
+++ b/algorithms-regression.md
@@ -168,7 +168,7 @@ y_test = lr.fit(df_train)
 
 
 
-### Arguments
+### Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) to read the matrix of feature vectors, each row
 constitutes one feature vector
@@ -570,7 +570,7 @@ lowest AIC is computed.
 
 
 
-### Arguments
+### Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) to read the matrix of feature vectors, each row
 contains one feature vector.
@@ -782,7 +782,7 @@ distributions and link functions, see below for details.
 
 
 
-### Arguments
+### Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) to read the matrix of feature vectors; each row
 constitutes an example.
@@ -1255,7 +1255,7 @@ distribution family is supported (see below for details).
 
 
 
-### Arguments
+### Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) to read the matrix of feature vectors; each row is an
 example.
@@ -1503,7 +1503,7 @@ this step outside the scope of `GLM-predict.dml` for now.
 
 
 
-### Arguments
+### Arguments for Spark and Hadoop invocation
 
 **X**: Location (on HDFS) to read the $n\,{\times}\,m$-matrix $X$ of feature
 vectors, each row constitutes one feature vector (one record)

[2/3] incubator-systemml git commit: [SYSTEMML-692] Added initial version of DML generator for Caffe

2017-04-19 Thread niketanpansare 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cc7993fc/src/main/proto/tensorflow/event.proto
--
diff --git a/src/main/proto/tensorflow/event.proto 
b/src/main/proto/tensorflow/event.proto
new file mode 100644
index 000..06d1992
--- /dev/null
+++ b/src/main/proto/tensorflow/event.proto
@@ -0,0 +1,102 @@
+//-
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+//-
+syntax = "proto3";
+
+package tensorflow;
+option cc_enable_arenas = true;
+option java_outer_classname = "EventProtos";
+option java_multiple_files = true;
+option java_package = "org.tensorflow.util";
+
+import "summary.proto";
+
+// Protocol buffer representing an event that happened during
+// the execution of a Brain model.
+message Event {
+  // Timestamp of the event.
+  double wall_time = 1;
+
+  // Global step of the event.
+  int64 step = 2;
+
+  oneof what {
+// An event file was started, with the specified version.
+// This is use to identify the contents of the record IO files
+// easily.  Current version is "brain.Event:2".  All versions
+// start with "brain.Event:".
+string file_version = 3;
+// An encoded version of a GraphDef.
+bytes graph_def = 4;
+// A summary was generated.
+Summary summary = 5;
+// The user output a log message. Not all messages are logged, only ones
+// generated via the Python tensorboard_logging module.
+LogMessage log_message = 6;
+// The state of the session which can be used for restarting after crashes.
+SessionLog session_log = 7;
+// The metadata returned by running a session.run() call.
+TaggedRunMetadata tagged_run_metadata = 8;
+// An encoded version of a MetaGraphDef.
+bytes meta_graph_def = 9;
+  }
+}
+
+// Protocol buffer used for logging messages to the events file.
+message LogMessage {
+  enum Level {
+UNKNOWN = 0;
+// Note: The logging level 10 cannot be named DEBUG. Some software
+// projects compile their C/C++ code with -DDEBUG in debug builds. So the
+// C++ code generated from this file should not have an identifier named
+// DEBUG.
+DEBUGGING = 10;
+INFO = 20;
+WARN = 30;
+ERROR = 40;
+FATAL = 50;
+  }
+  Level level = 1;
+  string message = 2;
+}
+
+// Protocol buffer used for logging session state.
+message SessionLog {
+  enum SessionStatus {
+STATUS_UNSPECIFIED = 0;
+START = 1;
+STOP = 2;
+CHECKPOINT = 3;
+  }
+
+  SessionStatus status = 1;
+  // This checkpoint_path contains both the path and filename.
+  string checkpoint_path = 2;
+  string msg = 3;
+}
+
+// For logging the metadata output for a single session.run() call.
+message TaggedRunMetadata {
+  // Tag name associated with this metadata.
+  string tag = 1;
+  // Byte-encoded version of the `RunMetadata` proto in order to allow lazy
+  // deserialization.
+  bytes run_metadata = 2;
+}

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cc7993fc/src/main/proto/tensorflow/summary.proto
--
diff --git a/src/main/proto/tensorflow/summary.proto 
b/src/main/proto/tensorflow/summary.proto
new file mode 100644
index 000..fc8053c
--- /dev/null
+++ b/src/main/proto/tensorflow/summary.proto
@@ -0,0 +1,123 @@
+//-
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express

[1/3] incubator-systemml git commit: [SYSTEMML-692] Added initial version of DML generator for Caffe

2017-04-19 Thread niketanpansare 
Repository: incubator-systemml
Updated Branches:
  refs/heads/master ad3e78a28 -> cc7993fc8


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/cc7993fc/src/main/scala/org/apache/sysml/api/dl/CaffeSolver.scala
--
diff --git a/src/main/scala/org/apache/sysml/api/dl/CaffeSolver.scala 
b/src/main/scala/org/apache/sysml/api/dl/CaffeSolver.scala
new file mode 100644
index 000..755949d
--- /dev/null
+++ b/src/main/scala/org/apache/sysml/api/dl/CaffeSolver.scala
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sysml.api.dl
+
+import caffe.Caffe.SolverParameter
+import org.apache.sysml.runtime.DMLRuntimeException
+import caffe.Caffe
+
+trait CaffeSolver {
+  def sourceFileName:String;
+  def update(dmlScript:StringBuilder, layer:CaffeLayer):Unit;
+  def init(dmlScript:StringBuilder, layer:CaffeLayer):Unit;
+  
+  // 
+  // Used for Fine-tuning
+  private def getLayerLr(layer:CaffeLayer, paramIndex:Int):String = {
+val param = layer.param.getParamList
+if(param == null || param.size() <= paramIndex)
+  return "lr"
+else
+  // TODO: Ignoring param.get(index).getDecayMult for now
+  return "(lr * " + param.get(paramIndex).getLrMult + ")"
+  }
+  // the first param { } is for the weights and the second is for the biases.
+  def getWeightLr(layer:CaffeLayer):String = getLayerLr(layer, 0)
+  def getBiasLr(layer:CaffeLayer):String = getLayerLr(layer, 1)
+  // 
+  
+  def commaSep(arr:String*):String = {
+ if(arr.length == 1) arr(0) else {
+   var ret = arr(0)
+   for(i <- 1 until arr.length) {
+ ret = ret + "," + arr(i)
+   }
+   ret
+ }
+   }
+  
+  def l2reg_update(lambda:Double, dmlScript:StringBuilder, 
layer:CaffeLayer):Unit = {
+// val donotRegularizeLayers:Boolean = layer.isInstanceOf[BatchNorm] || 
layer.isInstanceOf[Scale];
+if(lambda != 0 && layer.shouldUpdateWeight) {
+  dmlScript.append("\t").append(layer.dWeight + "_reg = l2_reg::backward(" 
+ layer.weight + ", " + lambda + ")\n")
+  dmlScript.append("\t").append(layer.dWeight + " = " + layer.dWeight + " 
+ " + layer.dWeight + "_reg\n")
+}
+  }
+}
+
+class LearningRatePolicy(lr_policy:String="exp", base_lr:Double=0.01) {
+  def this(solver:Caffe.SolverParameter) {
+this(solver.getLrPolicy, solver.getBaseLr)
+if(solver.hasGamma) setGamma(solver.getGamma)
+if(solver.hasStepsize) setStepsize(solver.getStepsize)
+if(solver.hasPower()) setPower(solver.getPower)
+  }
+  var gamma:Double = 0.95
+  var step:Double = 10
+  var power:Double = 0.75
+  def setGamma(gamma1:Double):Unit = { gamma = gamma1 } 
+  def setStepsize(step1:Double):Unit = { step = step1 } 
+  def setPower(power1:Double): Unit = { power = power1 }
+  def updateLearningRate(dmlScript:StringBuilder):Unit = {
+val new_lr = lr_policy.toLowerCase match {
+  case "fixed" => base_lr.toString
+  case "step" => "(" + base_lr + " * " +  gamma + " ^ " + " floor(e/" + 
step + "))"
+  case "exp" => "(" + base_lr + " * " + gamma + "^e)"
+  case "inv" =>  "(" + base_lr + "* (1 + " + gamma + " * e) ^ (-" + power 
+ "))"
+  case "poly" => "(" + base_lr  + " * (1 - e/ max_epochs) ^ " + power + ")"
+  case "sigmoid" => "(" + base_lr + "( 1/(1 + exp(-" + gamma + "* (e - " + 
step + ""
+  case _ => throw new DMLRuntimeException("The lr policy is not 
supported:" + lr_policy)
+}
+dmlScript.append("lr = " + new_lr + "\n")
+  }
+}
+
+/**
+ * lambda: regularization parameter
+ * momentum: Momentum value. Typical values are in the range of [0.5, 0.99], 
usually started at the lower end and annealed towards the higher end.
+ */
+class SGD(lambda:Double=5e-04, momentum:Double=0.9) extends CaffeSolver {
+  def update(dmlScript:StringBuilder, layer:CaffeLayer):Unit = {
+l2reg_update(lambda, dmlScript, layer)
+if(momentum == 0) {
+  // Use sgd
+  if(layer.shouldUpdateWeight) dmlScript.append("\t").append(layer.weight 
+ " =

svn commit: r19214 - /dev/incubator/systemml/0.14.0-incubating-rc4/

2017-04-19 Thread acs_s 
Author: acs_s
Date: Wed Apr 19 21:40:21 2017
New Revision: 19214

Log:
Apache SystemML 0.14.0-incubating-rc4

Added:
dev/incubator/systemml/0.14.0-incubating-rc4/

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz 
  (with props)

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.asc

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.md5

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.sha

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.zip 
  (with props)

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.zip.asc

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.zip.md5

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.zip.sha

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-python.tgz
   (with props)

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-python.tgz.asc

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-python.tgz.md5

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-python.tgz.sha

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-src.tgz 
  (with props)

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-src.tgz.asc

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-src.tgz.md5

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-src.tgz.sha

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-src.zip 
  (with props)

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-src.zip.asc

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-src.zip.md5

dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-src.zip.sha

Added: 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz
==
Binary file - no diff available.

Propchange: 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz
--
svn:mime-type = application/octet-stream

Added: 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.asc
==
--- 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.asc
 (added)
+++ 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.asc
 Wed Apr 19 21:40:21 2017
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2
+
+iQIcBAABCAAGBQJY99iIAAoJEEzdnt5TueFP8MkP/RlE9C3+KfTsOEkZP5+BcHbu
+wuJ8xEeQD3GQP2IjHWm3cn6qdlMBBYk/qwO2RuVpdU8Nn5ACfOgakvLijkJHYzDB
+JG474hlYlm67NwPyV9HfbzPl8OYB5/CC4mIOQbDQ0V+T5xD7ZRjForAC2RM/+gKf
+h4JMh3SlNuePjI0tcwfXRIlM4QWj3cg5o9WI1QSTuu8eEJdpvpjTf3JbR5LLsDGT
+PvQY2XKmyiMzlkmQSPSkAE/W7yb/gDrEKwpehIwoZzSbwixHBT4WvgF1iFPVhmES
+c3RE7kR+ElYJ3mwk/UFOa5tVzAmD3JHqrxge45WD0HcmIB/M8VGfEznqDxaN0+Pq
+/H35EuOzo509zi11VY1R5ByT1AJui7rQINRXeOnnIlXzmjzGlfmnDnryj5H+x5u7
+O3LaxOul2cOTmIjO3kQyuTzV7VkCxyUZD2fYiMW4UdvbxiYF8G2jChD3xeMPbweC
+Ue4/ntJ31V4N8M4+WBjhktsi8Anco9uc9Vzs7FKWJ3yXs2BbPXnn9VFDRFon7qBe
+Vyh0d+Matmeoru5/cQC/Z+rIGQq/xB8sqMbmVBUzP0GqBCrCgP55sWs/gvJnlEbh
+gdIi2TZxp+X3isVPPp6ZRtZ6ouMsp0f+6IeXu7dQjsqu1O3qn9wSK0NT1eU2cWwL
+nTXEGvTV6Jmwh4ez4S/N
+=7bmL
+-END PGP SIGNATURE-

Added: 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.md5
==
--- 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.md5
 (added)
+++ 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.md5
 Wed Apr 19 21:40:21 2017
@@ -0,0 +1 @@
+ddba1b87a4f4c8a435d961586103d444 *systemml-0.14.0-incubating-bin.tgz

Added: 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.sha
==
--- 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.sha
 (added)
+++ 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.tgz.sha
 Wed Apr 19 21:40:21 2017
@@ -0,0 +1 @@
+4fb3b46091534a9287beacdc1ae9b30084298543  systemml-0.14.0-incubating-bin.tgz

Added: 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.zip
==
Binary file - no diff available.

Propchange: 
dev/incubator/systemml/0.14.0-incubating-rc4/systemml-0.14.0-incubating-bin.zip
--

incubator-systemml git commit: [SYSTEMML-259] Function with no return value fix

2017-04-19 Thread deron 
Repository: incubator-systemml
Updated Branches:
  refs/heads/master ecdeff7d8 -> ad3e78a28


[SYSTEMML-259] Function with no return value fix

Add additional target null checks so that if a function does not return a
value and is not inlined, it does not require an lvalue.
Add MLContext tests with forced function calls.

Closes #463.


Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/ad3e78a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/ad3e78a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/ad3e78a2

Branch: refs/heads/master
Commit: ad3e78a288b543c626019de2af58e745469a1090
Parents: ecdeff7
Author: Deron Eriksson 
Authored: Wed Apr 19 14:37:58 2017 -0700
Committer: Deron Eriksson 
Committed: Wed Apr 19 14:37:58 2017 -0700

--
 .../org/apache/sysml/parser/DMLTranslator.java  | 14 +++---
 .../org/apache/sysml/parser/StatementBlock.java | 27 
 .../integration/mlcontext/MLContextTest.java| 20 +++
 3 files changed, 48 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ad3e78a2/src/main/java/org/apache/sysml/parser/DMLTranslator.java
--
diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java 
b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
index 6e4db6e..3373e98 100644
--- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java
+++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java
@@ -848,8 +848,10 @@ public class DMLTranslator
if (current instanceof AssignmentStatement) {
AssignmentStatement as = (AssignmentStatement) 
current;
DataIdentifier target = as.getTarget();
-   if (liveOut.containsVariable(target.getName())) 
{
-   liveOutToTemp.put(target.getName(), 
Integer.valueOf(i));
+   if (target != null) {
+   if 
(liveOut.containsVariable(target.getName())) {
+   
liveOutToTemp.put(target.getName(), Integer.valueOf(i));
+   }
}
}
if (current instanceof MultiAssignmentStatement) {
@@ -1065,7 +1067,13 @@ public class DMLTranslator
 
//create function op
FunctionType ftype = 
fsb.getFunctionOpType();
-   FunctionOp fcall = new 
FunctionOp(ftype, fci.getNamespace(), fci.getName(), finputs, new 
String[]{target.getName()});
+   FunctionOp fcall = null;
+   if (target == null) {
+   fcall = new FunctionOp(ftype, 
fci.getNamespace(), fci.getName(), finputs, new String[]{});
+   } else {
+   fcall = new FunctionOp(ftype, 
fci.getNamespace(), fci.getName(), finputs, new String[]{target.getName()});
+   }
+
output.add(fcall);

//TODO function output dataops (phase 3)

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ad3e78a2/src/main/java/org/apache/sysml/parser/StatementBlock.java
--
diff --git a/src/main/java/org/apache/sysml/parser/StatementBlock.java 
b/src/main/java/org/apache/sysml/parser/StatementBlock.java
index 8115166..7329ced 100644
--- a/src/main/java/org/apache/sysml/parser/StatementBlock.java
+++ b/src/main/java/org/apache/sysml/parser/StatementBlock.java
@@ -596,14 +596,16 @@ public class StatementBlock extends LiveVariableAnalysis
setStatementFormatType(as, conditional);

// Handle const vars: (a) basic constant 
propagation, and (b) transitive constant propagation over assignments 
-   currConstVars.remove(target.getName());
-   if(source instanceof ConstIdentifier && 
!(target instanceof IndexedIdentifier)){ //basic
-   currConstVars.put(target.getName(), 
(ConstIdentifier)source);
-   }
-   if( source instanceof DataIdentifier && 
!(target

incubator-systemml git commit: [maven-release-plugin] prepare for next development iteration

2017-04-19 Thread acs_s 
Repository: incubator-systemml
Updated Branches:
  refs/heads/master 8bdcf106c -> ecdeff7d8


[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/ecdeff7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/ecdeff7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/ecdeff7d

Branch: refs/heads/master
Commit: ecdeff7d8b45a6b62f9e8c957ab788f17773978f
Parents: 8bdcf10
Author: Arvind Surve 
Authored: Wed Apr 19 14:23:11 2017 -0700
Committer: Arvind Surve 
Committed: Wed Apr 19 14:23:11 2017 -0700

--
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ecdeff7d/pom.xml
--
diff --git a/pom.xml b/pom.xml
index e782f35..eba7f57 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
18

org.apache.systemml
-   0.14.0-incubating
+   1.0.0-incubating-SNAPSHOT
systemml
jar
SystemML
@@ -41,7 +41,7 @@

scm:git:g...@github.com:apache/incubator-systemml

scm:git:https://git-wip-us.apache.org/repos/asf/incubator-systemml

https://git-wip-us.apache.org/repos/asf?p=incubator-systemml.git
-   v0.14.0-incubating-rc4
+   HEAD


JIRA

[incubator-systemml] Git Push Summary

2017-04-19 Thread acs_s 
Repository: incubator-systemml
Updated Tags:  refs/tags/v0.14.0-incubating-rc4 [created] b633b1349

incubator-systemml git commit: [maven-release-plugin] prepare release v0.14.0-incubating-rc4

2017-04-19 Thread acs_s 
Repository: incubator-systemml
Updated Branches:
  refs/heads/master 7bf07252c -> 8bdcf106c


[maven-release-plugin] prepare release v0.14.0-incubating-rc4


Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/8bdcf106
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/8bdcf106
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/8bdcf106

Branch: refs/heads/master
Commit: 8bdcf106ca9bd04c0f68924ad5827eb7d7d54952
Parents: 7bf0725
Author: Arvind Surve 
Authored: Wed Apr 19 14:22:41 2017 -0700
Committer: Arvind Surve 
Committed: Wed Apr 19 14:22:41 2017 -0700

--
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/8bdcf106/pom.xml
--
diff --git a/pom.xml b/pom.xml
index e77c7d0..e782f35 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
18

org.apache.systemml
-   0.14.0-incubating-SNAPSHOT
+   0.14.0-incubating
systemml
jar
SystemML
@@ -41,7 +41,7 @@

scm:git:g...@github.com:apache/incubator-systemml

scm:git:https://git-wip-us.apache.org/repos/asf/incubator-systemml

https://git-wip-us.apache.org/repos/asf?p=incubator-systemml.git
-   HEAD
+   v0.14.0-incubating-rc4


JIRA

incubator-systemml git commit: [FIX] Setting release version number

2017-04-19 Thread acs_s 
Repository: incubator-systemml
Updated Branches:
  refs/heads/master 4c74a3434 -> 7bf07252c


[FIX] Setting release version number


Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7bf07252
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7bf07252
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7bf07252

Branch: refs/heads/master
Commit: 7bf07252c3873553e5f1a939d4e937319ddd7aba
Parents: 4c74a34
Author: Arvind Surve 
Authored: Wed Apr 19 12:46:36 2017 -0700
Committer: Arvind Surve 
Committed: Wed Apr 19 12:46:36 2017 -0700

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7bf07252/pom.xml
--
diff --git a/pom.xml b/pom.xml
index d761e24..e77c7d0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
18

org.apache.systemml
-   0.15.0-incubating-SNAPSHOT
+   0.14.0-incubating-SNAPSHOT
systemml
jar
SystemML

[2/2] incubator-systemml git commit: [SYSTEMML-1546] Fix parfor optimizer (result/task partitioning on spark)

2017-04-19 Thread mboehm7 
[SYSTEMML-1546] Fix parfor optimizer (result/task partitioning on spark)

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/4c74a343
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/4c74a343
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/4c74a343

Branch: refs/heads/master
Commit: 4c74a34349bd4eeb0f4e102db7bca1f09b2ced97
Parents: 30d4b1e
Author: Matthias Boehm 
Authored: Wed Apr 19 00:12:03 2017 -0700
Committer: Matthias Boehm 
Committed: Wed Apr 19 00:12:03 2017 -0700

--
 .../parfor/opt/OptimizerConstrained.java|  2 +-
 .../parfor/opt/OptimizerRuleBased.java  | 44 +++-
 2 files changed, 25 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4c74a343/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
--
diff --git 
a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
 
b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
index 235b927..fb83fd6 100644
--- 
a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
+++ 
b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java
@@ -131,7 +131,7 @@ public class OptimizerConstrained extends OptimizerRuleBased
boolean flagRecompMR = rewriteSetExecutionStategy( pn, M0a, M1, 
M2, M3, flagLIX );
 
//exec-type-specific rewrites
-   if( pn.getExecType() == ExecType.MR || pn.getExecType() == 
ExecType.SPARK )
+   if( pn.getExecType() == getRemoteExecType() )
{
if( M1 > _rm && M3 <= _rm  ) {
// rewrite 1: data partitioning (apply 
conditional partitioning)

http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/4c74a343/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
--
diff --git 
a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
 
b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
index f0eb0e7..0ff7a31 100644
--- 
a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
+++ 
b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java
@@ -262,7 +262,7 @@ public class OptimizerRuleBased extends Optimizer
boolean flagRecompMR = rewriteSetExecutionStategy( pn, M0a, M1, 
M2, M3, flagLIX );

//exec-type-specific rewrites
-   if( pn.getExecType() == ExecType.MR || 
pn.getExecType()==ExecType.SPARK )
+   if( pn.getExecType() == getRemoteExecType() )
{
if( M1 > _rm && M3 <= _rm  ) {
// rewrite 1: data partitioning (apply 
conditional partitioning)
@@ -400,6 +400,10 @@ public class OptimizerRuleBased extends Optimizer
_rkmax2  = (int) Math.ceil( PAR_K_FACTOR * _rk2 ); 
}

+   protected ExecType getRemoteExecType() {
+   return OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : 
ExecType.MR;
+   }
+   
///
//REWRITE set data partitioner
///
@@ -483,7 +487,7 @@ public class OptimizerRuleBased extends Optimizer
//NOTE: for the moment, we do not partition 
according to the remote mem, because we can execute 
//it even without partitioning in CP. However, 
advanced optimizers should reason about this

//double mold = h.getMemEstimate();
-   if(n.getExecType() == ExecType.MR ||  
n.getExecType()==ExecType.SPARK  //Opt Condition: MR/Spark
+   if(n.getExecType() == 
getRemoteExecType()  //Opt Condition: MR/Spark
|| h.getMemEstimate() > thetaM ) //Opt 
Condition: mem estimate > constraint to force partitioning   
{
//NOTE: subsequent rewrites will still 
use the MR mem estimate
@@ -608,23 +612,22 @@ public class OptimizerRuleBased extends Optimizer
ParForProgramBlock pfpb = (ParForProgramBlock) o[1];

//search for candidates
-   Collection cand =

[1/2] incubator-systemml git commit: [SYSTEMML-1545] Fix memory estimates leftindexing w/ empty inputs

2017-04-19 Thread mboehm7 
Repository: incubator-systemml
Updated Branches:
  refs/heads/master cd5499c54 -> 4c74a3434


[SYSTEMML-1545] Fix memory estimates leftindexing w/ empty inputs

For a detailed problem description see
https://issues.apache.org/jira/browse/SYSTEMML-1545

This patch overcomes the incorrect memory estimates by only considering
the input nnz as output sparsity for empty left-hand-side matrices if
the indexing identifiers are constant, e.g., during dynamic
recompilation (after literal replacement); otherwise the sparsity is
left as unknown. 

Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: 
http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/30d4b1ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/30d4b1ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/30d4b1ee

Branch: refs/heads/master
Commit: 30d4b1eef85509c0f0cf646b15e4946d0eed3d32
Parents: cd5499c
Author: Matthias Boehm 
Authored: Wed Apr 19 00:10:03 2017 -0700
Committer: Matthias Boehm 
Committed: Wed Apr 19 00:10:03 2017 -0700

--
 .../org/apache/sysml/hops/LeftIndexingOp.java   | 33 +++-
 .../recompile/SparsityRecompileTest.java|  8 ++---
 2 files changed, 23 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/30d4b1ee/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
--
diff --git a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java 
b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
index 89c025b..f927af7 100644
--- a/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/LeftIndexingOp.java
@@ -113,15 +113,7 @@ public class LeftIndexingOp  extends Hop
Lop bottom=getInput().get(3).constructLops();
Lop left=getInput().get(4).constructLops();
Lop right=getInput().get(5).constructLops();
-   /*
-   //need to creat new lops for converting the 
index ranges
-   //original range is (a, b) --> (c, d)
-   //newa=2-a, newb=2-b
-   Lops two=new Data(null, 
Data.OperationTypes.READ, null, "2", Expression.DataType.SCALAR, 
Expression.ValueType.INT, false);
-   Lops newTop=new Binary(two, top, 
HopsOpOp2LopsB.get(Hops.OpOp2.MINUS), Expression.DataType.SCALAR, 
Expression.ValueType.INT, et);
-   Lops newLeft=new Binary(two, left, 
HopsOpOp2LopsB.get(Hops.OpOp2.MINUS), Expression.DataType.SCALAR, 
Expression.ValueType.INT, et);
-   //newc=leftmatrix.row-a+1, newd=leftmatrix.row
-   */
+   
//right hand matrix
Lop nrow=new 
UnaryCP(getInput().get(0).constructLops(), 

OperationTypes.NROW, DataType.SCALAR, ValueType.INT);
@@ -298,7 +290,9 @@ public class LeftIndexingOp  extends Hop
//(this is important for indexing sparse matrices into 
empty matrices).
MatrixCharacteristics mcM1 = 
memo.getAllInputStats(getInput().get(0));
MatrixCharacteristics mcM2 = 
memo.getAllInputStats(getInput().get(1));
-   if( mcM1.getNonZeros()>=0 && mcM2.getNonZeros()>=0  ) {
+   if( mcM1.getNonZeros()>=0 && mcM2.getNonZeros()>=0
+   && hasConstantIndexingRange() ) 
+   {
long lnnz = mcM1.getNonZeros() + 
mcM2.getNonZeros();
_outputMemEstimate = computeOutputMemEstimate( 
_dim1, _dim2, lnnz );
_memEstimate = getInputSize(0) //original 
matrix (left)
@@ -316,7 +310,7 @@ public class LeftIndexingOp  extends Hop
{
Hop input1 = getInput().get(0);
Hop input2 = getInput().get(1);
-   if( input1.dimsKnown() ) {
+   if( input1.dimsKnown() && hasConstantIndexingRange() ) {
sparsity = 
OptimizerUtils.getLeftIndexingSparsity(
input1.getDim1(), 
input1.getDim2(), input1.getNnz(), 
input2.getDim1(), 
input2.getDim2(), input2.getNnz());
@@ -351,8 +345,8 @@ public class LeftIndexingOp  extends Hop
double sparsity = 
OptimizerUtils.getLeftIndexingSparsity(