[1/2] systemml git commit: [MINOR] Rename rangeReIndex to rightIndex and related cleanups

[mboehm7]

Repository: systemml
Updated Branches:
  refs/heads/master d7f765331 -> d5f20b43b


[MINOR] Rename rangeReIndex to rightIndex and related cleanups

This patch renames the rangeReIndex to rightIndex to make it consistent
with the existing leftIndex. Furthermore, this also centralizes these
left/right indexing opcodes and modifies their usage accordingly. 


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/e60a4c29
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/e60a4c29
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/e60a4c29

Branch: refs/heads/master
Commit: e60a4c29dd1e6f3ccdcee8cae27628b75c2d4742
Parents: d7f7653
Author: Matthias Boehm 
Authored: Tue Sep 5 12:29:29 2017 -0700
Committer: Matthias Boehm 
Committed: Tue Sep 5 21:18:09 2017 -0700

--
 .../java/org/apache/sysml/hops/IndexingOp.java  |   8 +-
 .../org/apache/sysml/hops/LeftIndexingOp.java   |   4 +-
 .../hops/cost/CostEstimatorStaticRuntime.java   |  14 +-
 .../java/org/apache/sysml/lops/LeftIndex.java   |   6 +-
 src/main/java/org/apache/sysml/lops/Lop.java|  20 +-
 .../apache/sysml/lops/RangeBasedReIndex.java| 191 ---
 .../java/org/apache/sysml/lops/RightIndex.java  | 188 ++
 .../instructions/CPInstructionParser.java   |   6 +-
 .../instructions/GPUInstructionParser.java  |   4 +-
 .../instructions/MRInstructionParser.java   |  11 +-
 .../instructions/SPInstructionParser.java   |   8 +-
 .../cp/FrameIndexingCPInstruction.java  |   6 +-
 .../instructions/cp/IndexingCPInstruction.java  |  47 ++---
 .../cp/MatrixIndexingCPInstruction.java |   6 +-
 .../cpfile/MatrixIndexingCPFileInstruction.java |  29 ++-
 .../gpu/MatrixIndexingGPUInstruction.java   |  28 ++-
 .../runtime/instructions/mr/MRInstruction.java  |   2 +-
 .../mr/RangeBasedReIndexInstruction.java|   7 +-
 .../spark/FrameIndexingSPInstruction.java   |  17 +-
 .../spark/IndexingSPInstruction.java|  18 +-
 .../spark/MatrixIndexingSPInstruction.java  |  18 +-
 .../runtime/matrix/data/LibMatrixCUDA.java  |   2 +-
 .../sysml/runtime/matrix/data/MatrixBlock.java  |   2 +-
 .../functions/codegen/RowAggTmplTest.java   |   5 +-
 .../recompile/RemoveEmptyRecompileTest.java |  12 +-
 25 files changed, 312 insertions(+), 347 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/e60a4c29/src/main/java/org/apache/sysml/hops/IndexingOp.java
--
diff --git a/src/main/java/org/apache/sysml/hops/IndexingOp.java 
b/src/main/java/org/apache/sysml/hops/IndexingOp.java
index 6a3ddf4..2463c7d 100644
--- a/src/main/java/org/apache/sysml/hops/IndexingOp.java
+++ b/src/main/java/org/apache/sysml/hops/IndexingOp.java
@@ -27,7 +27,7 @@ import org.apache.sysml.lops.Data;
 import org.apache.sysml.lops.Group;
 import org.apache.sysml.lops.Lop;
 import org.apache.sysml.lops.LopsException;
-import org.apache.sysml.lops.RangeBasedReIndex;
+import org.apache.sysml.lops.RightIndex;
 import org.apache.sysml.lops.LopProperties.ExecType;
 import org.apache.sysml.parser.Expression.DataType;
 import org.apache.sysml.parser.Expression.ValueType;
@@ -134,7 +134,7 @@ public class IndexingOp extends Hop

   input._dim1, input._dim2, _dim1, _dim2);

Lop dummy = 
Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
-   RangeBasedReIndex reindex = new 
RangeBasedReIndex(
+   RightIndex reindex = new RightIndex(
input.constructLops(), 
getInput().get(1).constructLops(), getInput().get(2).constructLops(),

getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, 
dummy,
getDataType(), 
getValueType(), et);
@@ -170,7 +170,7 @@ public class IndexingOp extends Hop
SparkAggType.NONE : 
SparkAggType.MULTI_BLOCK;

Lop dummy = 
Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
-   RangeBasedReIndex reindex = new 
RangeBasedReIndex(
+   RightIndex reindex = new RightIndex(
input.constructLops(), 
getInput().get(1).constructLops(), getInput().get(2).constructLops(),
 

[2/2] systemml git commit: [SYSTEMML-1889] Robustness parfor/hop memory budgets in spark exec mode

[mboehm7]

[SYSTEMML-1889] Robustness parfor/hop memory budgets in spark exec mode

Recently, we encountered robustness issues of parfor memory budgets in
spark execution mode (remote memory set to Infinity). This patch
improves the robustness of analyzing the spark cluster configuration,
and the parfor-specific infrastructure analysis. The encountered issue
was likely due to an invalid default parallelism (less than the number
of executors), leading to MEM/(dpar/#exec)=INF. 

Furthermore, this patch also updates the internal defaults of min and
max storage memory fractions to the defaults used in Spark 2.1 and 2.2.
Additionally, parfor now also reports the entire Spark cluster config
when log=DEBUG, without unnecessarily creating the Spark context.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d5f20b43
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d5f20b43
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d5f20b43

Branch: refs/heads/master
Commit: d5f20b43b9e74e73747ee65f11f9767577b001e8
Parents: e60a4c2
Author: Matthias Boehm 
Authored: Tue Sep 5 17:57:06 2017 -0700
Committer: Matthias Boehm 
Committed: Tue Sep 5 21:18:10 2017 -0700

--
 .../context/SparkExecutionContext.java  | 28 +++-
 .../parfor/opt/OptimizerRuleBased.java  | 12 ++---
 2 files changed, 24 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/d5f20b43/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
--
diff --git 
a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
 
b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
index d1ff7d8..e0352b0 100644
--- 
a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
+++ 
b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java
@@ -1382,7 +1382,7 @@ public class SparkExecutionContext extends 
ExecutionContext
{
//broadcasts are stored in mem-and-disk in data space, this 
config
//defines the fraction of data space to be used as broadcast 
budget
-   private static final double BROADCAST_DATA_FRACTION = 0.3;
+   private static final double BROADCAST_DATA_FRACTION = 0.35;
 
//forward private config from Spark's 
UnifiedMemoryManager.scala (>1.6)
private static final long RESERVED_SYSTEM_MEMORY_BYTES = 300 * 
1024 * 1024;
@@ -1430,7 +1430,7 @@ public class SparkExecutionContext extends 
ExecutionContext
//always get the current num executors on refresh 
because this might
//change if not all executors are initially allocated 
and it is plan-relevant
int numExec = _numExecutors;
-   if( refresh && !_confOnly ) {
+   if( (refresh && !_confOnly) || isSparkContextCreated() 
) {
JavaSparkContext jsc = getSparkContextStatic();
numExec = 
Math.max(jsc.sc().getExecutorMemoryStatus().size() - 1, 1);
}
@@ -1452,14 +1452,15 @@ public class SparkExecutionContext extends 
ExecutionContext
 
//always get the current default parallelism on refresh 
because this might
//change if not all executors are initially allocated 
and it is plan-relevant
-   return ( refresh && !_confOnly ) ?
+   int par = ( (refresh && !_confOnly) || 
isSparkContextCreated() ) ?
getSparkContextStatic().defaultParallelism() : 
_defaultPar;
+   return Math.max(par, 1); //robustness min parallelism
}
 
public void analyzeSparkConfiguationLegacy(SparkConf conf)  {
//ensure allocated spark conf
SparkConf sconf = (conf == null) ? 
createSystemMLSparkConf() : conf;
-
+   
//parse absolute executor memory
_memExecutor = UtilFunctions.parseMemorySize(
sconf.get("spark.executor.memory", 
"1g"));
@@ -1477,7 +1478,7 @@ public class SparkExecutionContext extends 
ExecutionContext
public void analyzeSparkConfiguation(SparkConf conf) {
//ensure allocated spark conf
SparkConf sconf = (conf == null) ? 
createSystemMLSparkConf() : conf;
-
+   
//parse 

[systemml] Git Push Summary

[acs_s]

Repository: systemml
Updated Branches:
  refs/heads/branch-0.15 [created] 467de1cb1

svn commit: r21466 - /dev/systemml/0.15.0-rc1/

[acs_s]

Author: acs_s
Date: Tue Sep  5 18:37:18 2017
New Revision: 21466

Log:
Apache SystemML 0.15.0-rc1

Added:
dev/systemml/0.15.0-rc1/
dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz   (with props)
dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.asc
dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.md5
dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.sha512
dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.zip   (with props)
dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.zip.asc
dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.zip.md5
dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.zip.sha512
dev/systemml/0.15.0-rc1/systemml-0.15.0-python.tgz   (with props)
dev/systemml/0.15.0-rc1/systemml-0.15.0-python.tgz.asc
dev/systemml/0.15.0-rc1/systemml-0.15.0-python.tgz.md5
dev/systemml/0.15.0-rc1/systemml-0.15.0-python.tgz.sha512
dev/systemml/0.15.0-rc1/systemml-0.15.0-src.tgz   (with props)
dev/systemml/0.15.0-rc1/systemml-0.15.0-src.tgz.asc
dev/systemml/0.15.0-rc1/systemml-0.15.0-src.tgz.md5
dev/systemml/0.15.0-rc1/systemml-0.15.0-src.tgz.sha512
dev/systemml/0.15.0-rc1/systemml-0.15.0-src.zip   (with props)
dev/systemml/0.15.0-rc1/systemml-0.15.0-src.zip.asc
dev/systemml/0.15.0-rc1/systemml-0.15.0-src.zip.md5
dev/systemml/0.15.0-rc1/systemml-0.15.0-src.zip.sha512

Added: dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz
==
Binary file - no diff available.

Propchange: dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz
--
svn:mime-type = application/octet-stream

Added: dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.asc
==
--- dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.asc (added)
+++ dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.asc Tue Sep  5 18:37:18 2017
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2
+
+iQIcBAABCAAGBQJZru3rAAoJEEzdnt5TueFPZu8P/0lDMQxjwtbW7QJ39CGWbWvk
+H2siG/lLk0fiuLwterx+InqzxnA9JThs6BiIS2EPOIRWMEI8h64HZYeFosbs4r3U
+laPOjrVPgJd4mmBkYnz08NsPELEf50DRCmeEWwswCWcaFaj5/OwPQYuN4cBsmVVs
+GlZxy45a/3iqnYHHlYV9TYpj845IvuyfMnSoKkSaH6aZ9yw9EKMQmJoYuht6lqVY
+X+2RvgF699UZIeD3gmeD37H2T2T5U8cvljZS7saiMyfZSqIvUUy3vJwCKazW3hbe
+k2Aej/k/GQkae4sJk4tlGf7pzgm7GJ6xS32/NRJYOw6YOzi6NE6YCqa3iFGcHb80
+cegkPL9pW6Gg/mexTv05G+eptcwdCDmot5c78yEkmZHJgHWM4XTCFC3btTX/ASWE
+ZK1hLt6tOogLxW+ieN0JXvGksHVNrZuxQDCMlypL3kph6+VzoUPu9oO5J8MapoFO
+GxLNB977L6m9xiaJ5KZ9uULiti8vo5v6WKFwRXRgMT3s50loXLz3W/I8F7oN6sOG
+vIBoYrXptRl68S3TBHl8uT+eu/Agcsujtqmepj2h+BU3GcYrC5pbX/sUPS5D2HhJ
+FjKvg4oeRC+3vPrnWrhHu9ABg53YoPqzox9XQ1/kHQZbPB2nD+S8ZyaFpz2+ClXx
+ljkcfKuAXse0QBxGrKXb
+=0CRn
+-END PGP SIGNATURE-

Added: dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.md5
==
--- dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.md5 (added)
+++ dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.md5 Tue Sep  5 18:37:18 2017
@@ -0,0 +1 @@
+f2e1d99c93e49231f2bba50f4d9cb3e8 *systemml-0.15.0-bin.tgz

Added: dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.sha512
==
--- dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.sha512 (added)
+++ dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.tgz.sha512 Tue Sep  5 18:37:18 
2017
@@ -0,0 +1 @@
+a97cd31c2c74b2874e913198e3b4b7486313224a1de6c4057b7ae36c9317111b7c18ef3d5b940f6280ec5d795f6b95e503fa05f9173f9dea408afa4e29f8e94b
  systemml-0.15.0-bin.tgz

Added: dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.zip
==
Binary file - no diff available.

Propchange: dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.zip
--
svn:mime-type = application/octet-stream

Added: dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.zip.asc
==
--- dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.zip.asc (added)
+++ dev/systemml/0.15.0-rc1/systemml-0.15.0-bin.zip.asc Tue Sep  5 18:37:18 2017
@@ -0,0 +1,17 @@
+-BEGIN PGP SIGNATURE-
+Version: GnuPG v2
+
+iQIcBAABCAAGBQJZru3eAAoJEEzdnt5TueFPpzwQAIZlj69cFH+LKU0CY0PGIilR
+3eWGqwD1oQrP9dt3MNZYzWBjVxX5i+3zOeNYB49yLPDEWd95GWePSmXbQHD3bUUN
+mlMp0YP9Q/JNLAGK9XEnnDC0K4Qt8d5PIfJV5rUEthT/e8GI8nsiFIW+3OREO9aI
+E9viLjyBLz9nUnDLvh9grzPakz8sM4kgwLp5JEkfbKRGM0598YIjillMVIXfzQES
+QdAsxPcAHZUqslZT1jxt6IF+nLsFZfukJfeP5wzDEeNV9drR8qamJHZU1ROjfpZu
+tMZI5/yCXg8q1KI2RV4AZKMMh1q6nOizyPHExGzIolWsFRedxPauaqE/0XowmBck
+r2qxvlSUEZPcHXiJoWjZ0f7Z+Ibj269zWEKKtaok9EzuqA4nuzeEke2cTjs0lCD4
+MSJSBjpb9P/YAAQrVOs1MYYqlnAoMfjoIonOyJzPpEFtOBeuc2FIaszdFCjH9Om5
+usARzVGGtvkmkuK9qmHVnCqckBHvGNF9Wgu1+sepjNuIC8qCX3NSkbyxZSHD5cic

systemml git commit: [maven-release-plugin] prepare for next development iteration

[acs_s]

Repository: systemml
Updated Branches:
  refs/heads/master 467de1cb1 -> d7f765331


[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d7f76533
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d7f76533
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d7f76533

Branch: refs/heads/master
Commit: d7f7653319eadcc71a7617389fb28e4e08ffd099
Parents: 467de1c
Author: Arvind Surve 
Authored: Tue Sep 5 10:46:07 2017 -0700
Committer: Arvind Surve 
Committed: Tue Sep 5 10:46:07 2017 -0700

--
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/d7f76533/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 9b57ba8..4bbe714 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
18

org.apache.systemml
-   0.15.0
+   1.0.0-SNAPSHOT
systemml
jar
SystemML
@@ -41,7 +41,7 @@
scm:git:g...@github.com:apache/systemml

scm:git:https://git-wip-us.apache.org/repos/asf/systemml

https://git-wip-us.apache.org/repos/asf?p=systemml.git
-   v0.15.0-rc1
+   HEAD


JIRA

[systemml] Git Push Summary

[acs_s]

Repository: systemml
Updated Tags:  refs/tags/v0.15.0-rc1 [created] fdb2be56a

systemml git commit: [maven-release-plugin] prepare release v0.15.0-rc1

[acs_s]

Repository: systemml
Updated Branches:
  refs/heads/master ade8df5d4 -> 467de1cb1


[maven-release-plugin] prepare release v0.15.0-rc1


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/467de1cb
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/467de1cb
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/467de1cb

Branch: refs/heads/master
Commit: 467de1cb17c70a1f9796e732d34cb2c216bb70a8
Parents: ade8df5
Author: Arvind Surve 
Authored: Tue Sep 5 10:45:51 2017 -0700
Committer: Arvind Surve 
Committed: Tue Sep 5 10:45:51 2017 -0700

--
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/467de1cb/pom.xml
--
diff --git a/pom.xml b/pom.xml
index eb6f026..9b57ba8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
18

org.apache.systemml
-   1.0.0-incubating-SNAPSHOT
+   0.15.0
systemml
jar
SystemML
@@ -41,7 +41,7 @@
scm:git:g...@github.com:apache/systemml

scm:git:https://git-wip-us.apache.org/repos/asf/systemml

https://git-wip-us.apache.org/repos/asf?p=systemml.git
-   HEAD
+   v0.15.0-rc1


JIRA

[systemml] Git Push Summary

[acs_s]

Repository: systemml
Updated Tags:  refs/tags/v0.15.0-incubating-rc1 [deleted] 75f700ef7

systemml git commit: [maven-release-plugin] prepare for next development iteration

[acs_s]

Repository: systemml
Updated Branches:
  refs/heads/master a073ad24a -> ade8df5d4


[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/ade8df5d
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/ade8df5d
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/ade8df5d

Branch: refs/heads/master
Commit: ade8df5d41cc60808e9bcb9fb48ff67bdf91f153
Parents: a073ad2
Author: Arvind Surve 
Authored: Tue Sep 5 10:26:32 2017 -0700
Committer: Arvind Surve 
Committed: Tue Sep 5 10:26:32 2017 -0700

--
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/ade8df5d/pom.xml
--
diff --git a/pom.xml b/pom.xml
index dc0af9c..eb6f026 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
18

org.apache.systemml
-   0.15.0-incubating
+   1.0.0-incubating-SNAPSHOT
systemml
jar
SystemML
@@ -41,7 +41,7 @@
scm:git:g...@github.com:apache/systemml

scm:git:https://git-wip-us.apache.org/repos/asf/systemml

https://git-wip-us.apache.org/repos/asf?p=systemml.git
-   v0.15.0-incubating-rc1
+   HEAD


JIRA

systemml git commit: [maven-release-plugin] prepare release v0.15.0-incubating-rc1

[acs_s]

Repository: systemml
Updated Branches:
  refs/heads/master b34079a28 -> a073ad24a


[maven-release-plugin] prepare release v0.15.0-incubating-rc1


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/a073ad24
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/a073ad24
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/a073ad24

Branch: refs/heads/master
Commit: a073ad24a3ab1cdbdc77a252e8f72e1d45037fc3
Parents: b34079a
Author: Arvind Surve 
Authored: Tue Sep 5 10:26:14 2017 -0700
Committer: Arvind Surve 
Committed: Tue Sep 5 10:26:14 2017 -0700

--
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/a073ad24/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 4bbe714..dc0af9c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
18

org.apache.systemml
-   1.0.0-SNAPSHOT
+   0.15.0-incubating
systemml
jar
SystemML
@@ -41,7 +41,7 @@
scm:git:g...@github.com:apache/systemml

scm:git:https://git-wip-us.apache.org/repos/asf/systemml

https://git-wip-us.apache.org/repos/asf?p=systemml.git
-   HEAD
+   v0.15.0-incubating-rc1


JIRA

[systemml] Git Push Summary

[acs_s]

Repository: systemml
Updated Tags:  refs/tags/v0.15.0-incubating-rc1 [created] 75f700ef7

systemml git commit: [MINOR] Enable systemml to be imported in the pyspark workers

[niketanpansare]

Repository: systemml
Updated Branches:
  refs/heads/master 0ba9e74b9 -> b34079a28


[MINOR] Enable systemml to be imported in the pyspark workers

- Moved the race condition avoidance logic to  classloader instead of
  import level. This avoids creation of dataframe in pyspark workers.

Closes #652.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/b34079a2
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/b34079a2
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/b34079a2

Branch: refs/heads/master
Commit: b34079a283a1859ed23de77f4ff0e50985b57dd3
Parents: 0ba9e74
Author: Niketan Pansare 
Authored: Tue Sep 5 09:59:59 2017 -0700
Committer: Niketan Pansare 
Committed: Tue Sep 5 10:01:55 2017 -0700

--
 src/main/python/systemml/classloader.py | 14 +-
 src/main/python/systemml/mlcontext.py   |  4 
 2 files changed, 13 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/b34079a2/src/main/python/systemml/classloader.py
--
diff --git a/src/main/python/systemml/classloader.py 
b/src/main/python/systemml/classloader.py
index 8738dc5..015a3dc 100644
--- a/src/main/python/systemml/classloader.py
+++ b/src/main/python/systemml/classloader.py
@@ -22,15 +22,27 @@
 __all__ = ['createJavaObject']
 
 import os
+import numpy as np
+import pandas as pd
 
 try:
 import py4j.java_gateway
 from py4j.java_gateway import JavaObject
 from pyspark import SparkContext
+from pyspark.sql import SparkSession
 except ImportError:
 raise ImportError('Unable to import `pyspark`. Hint: Make sure you are 
running with PySpark.')
 
+_initializedSparkSession = False
 def _createJavaObject(sc, obj_type):
+# 
---
+# Avoids race condition between locking of metastore_db of Scala 
SparkSession and PySpark SparkSession.
+# This is done at toDF() rather than import level to avoid creation of 
SparkSession in worker processes.
+global _initializedSparkSession
+if not _initializedSparkSession:
+_initializedSparkSession = True
+
SparkSession.builder.getOrCreate().createDataFrame(pd.DataFrame(np.array([[1,2],[3,4]])))
+# 
---
 if obj_type == 'mlcontext':
 return sc._jvm.org.apache.sysml.api.mlcontext.MLContext(sc._jsc)
 elif obj_type == 'dummy':
@@ -89,4 +101,4 @@ def createJavaObject(sc, obj_type):
 jar_file_name = _getJarFileName(sc, '-extra')
 x = _getLoaderInstance(sc, jar_file_name, 
'org.apache.sysml.api.dl.Caffe2DMLLoader', hint + 'systemml-*-extra.jar')
 x.loadCaffe2DML(jar_file_name)
-return ret
\ No newline at end of file
+return ret

http://git-wip-us.apache.org/repos/asf/systemml/blob/b34079a2/src/main/python/systemml/mlcontext.py
--
diff --git a/src/main/python/systemml/mlcontext.py 
b/src/main/python/systemml/mlcontext.py
index 60705c5..4a555f7 100644
--- a/src/main/python/systemml/mlcontext.py
+++ b/src/main/python/systemml/mlcontext.py
@@ -36,11 +36,7 @@ try:
 from pyspark import SparkContext
 from pyspark.conf import SparkConf
 import pyspark.mllib.common
-# 
---
-# Avoids race condition between locking of metastore_db of Scala 
SparkSession and PySpark SparkSession
 from pyspark.sql import SparkSession
-
SparkSession.builder.getOrCreate().createDataFrame(pd.DataFrame(np.array([[1,2],[3,4]])))
-# 
---
 except ImportError:
 raise ImportError('Unable to import `pyspark`. Hint: Make sure you are 
running with PySpark.')
 

systemml git commit: [SYSTEMML-1890] Update Release Distribution policy

[acs_s]

Repository: systemml
Updated Branches:
  refs/heads/master 1cbfdef3f -> 0ba9e74b9


[SYSTEMML-1890] Update Release Distribution policy


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/0ba9e74b
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/0ba9e74b
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/0ba9e74b

Branch: refs/heads/master
Commit: 0ba9e74b96d2638e0ca2d9af569cdcc26676a78e
Parents: 1cbfdef
Author: Arvind Surve 
Authored: Tue Sep 5 09:59:46 2017 -0700
Committer: Arvind Surve 
Committed: Tue Sep 5 09:59:46 2017 -0700

--
 dev/release/release-build.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/0ba9e74b/dev/release/release-build.sh
--
diff --git a/dev/release/release-build.sh b/dev/release/release-build.sh
index 040ec36..66e0a88 100755
--- a/dev/release/release-build.sh
+++ b/dev/release/release-build.sh
@@ -289,8 +289,8 @@ if [[ "$RELEASE_PREPARE" == "true" ]]; then
 for i in *.zip *.tgz; do gpg --output $i.asc --detach-sig --armor $i; 
done
 rm -f *.md5
 for i in *.zip *.tgz; do openssl md5 -hex $i | sed 's/MD5(\([^)]*\))= 
\([0-9a-f]*\)/\2 *\1/' > $i.md5; done
-rm -f *.sha
-for i in *.zip *.tgz; do shasum $i > $i.sha; done
+rm -f *.sha512
+for i in *.zip *.tgz; do shasum -a 512 $i > $i.sha512; done
 
 cd .. #exit $RELEASE_VERSION-$RELEASE_RC/
 

[1/2] systemml git commit: [SYSTEMML-1886] Extended codegen outer template (sparse-safe driver ops)

[mboehm7]

Repository: systemml
Updated Branches:
  refs/heads/master 3f3e927b8 -> 1cbfdef3f


[SYSTEMML-1886] Extended codegen outer template (sparse-safe driver ops)

This patch extends the codegen outer template by the ability to fuse
additional sparse-safe operations on the driver. In detail, this is
realized by generalizing the OFMC-merge condition of the outer template
as well as the related cplan construction.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/30caf365
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/30caf365
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/30caf365

Branch: refs/heads/master
Commit: 30caf3652125851979b3bca3312f98551853fbed
Parents: 3f3e927
Author: Matthias Boehm 
Authored: Mon Sep 4 19:08:27 2017 -0700
Committer: Matthias Boehm 
Committed: Mon Sep 4 19:08:27 2017 -0700

--
 .../org/apache/sysml/hops/OptimizerUtils.java   | 10 ---
 .../codegen/template/TemplateOuterProduct.java  | 13 
 .../hops/codegen/template/TemplateUtils.java| 18 
 .../sysml/hops/rewrite/HopRewriteUtils.java | 13 
 .../functions/codegen/OuterProdTmplTest.java| 27 ++---
 src/test/scripts/functions/codegen/wdivmmNeq.R  | 31 
 .../scripts/functions/codegen/wdivmmNeq.dml | 29 ++
 7 files changed, 128 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/30caf365/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
--
diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java 
b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
index a4143e3..82b1848 100644
--- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
+++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java
@@ -1015,15 +1015,17 @@ public class OptimizerUtils
return ( op==OpOp2.NOTEQUAL && val==0);
}
 
-   public static double getBinaryOpSparsityConditionalSparseSafe( double 
sp1, OpOp2 op, LiteralOp lit )
-   {
+   public static boolean isBinaryOpSparsityConditionalSparseSafe( OpOp2 
op, LiteralOp lit ) {
double val = HopRewriteUtils.getDoubleValueSafe(lit);
-   
return (  (op==OpOp2.GREATER  && val==0) 
||(op==OpOp2.LESS && val==0)
||(op==OpOp2.NOTEQUAL && val==0)
||(op==OpOp2.EQUAL&& val!=0)
-   ||(op==OpOp2.MINUS&& val==0)) ? sp1 : 1.0;
+   ||(op==OpOp2.MINUS&& val==0));
+   }
+   
+   public static double getBinaryOpSparsityConditionalSparseSafe( double 
sp1, OpOp2 op, LiteralOp lit ) {
+   return isBinaryOpSparsityConditionalSparseSafe(op, lit) ? sp1 : 
1.0;
}

/**

http://git-wip-us.apache.org/repos/asf/systemml/blob/30caf365/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java
--
diff --git 
a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java
 
b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java
index f001a81..ec2ee3b 100644
--- 
a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java
+++ 
b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java
@@ -78,7 +78,10 @@ public class TemplateOuterProduct extends TemplateBase {
public boolean merge(Hop hop, Hop input) {
return !isClosed() && 
(TemplateUtils.isBinaryMatrixRowVector(hop)
-   || HopRewriteUtils.isBinaryMatrixScalarOperation(hop));
+   || HopRewriteUtils.isBinaryMatrixScalarOperation(hop)
+   || (HopRewriteUtils.isBinary(hop, OpOp2.MULT) 
+   && HopRewriteUtils.isBinarySparseSafe(input)
+   && 
!TemplateUtils.rContainsOuterProduct(input)));
}
 
@Override
@@ -167,10 +170,10 @@ public class TemplateOuterProduct extends TemplateBase {
CNode cdata2 = 
tmp.get(hop.getInput().get(1).getHopID());
String primitiveOpName = 
((BinaryOp)hop).getOp().toString();

-   if( HopRewriteUtils.isEqualSize(hop.getInput().get(0), 
hop.getInput().get(1)) ) {
-   Hop main = hop.getInput().get((cdata1 
instanceof CNodeData) ? 0 : 1);
-   inHops2.put("_X", main);
-   }
+   

[2/2] systemml git commit: [SYSTEMML-1888] Remove parfor rewrite on update-in-place intermediates

[mboehm7]

[SYSTEMML-1888] Remove parfor rewrite on update-in-place intermediates

This patch removes the brittle parfor rewrite on update-in-place
intermediates that has shown to blow up for large parfor programs and
many candidates due to poor asymptotic behavior (at least squared in the
number of nodes and candidates, large constants). Furthermore, this
rewrites also relies on comparisons of hop names and line numbers to
check validity, which is dangerous in the presence of other rewrites. 

Meanwhile, we apply anyway a very robust update-in-place rewrite to all
for and parfor loops, and hence, the performance impact of this parfor
rewrite is rather limited. Therefore, we remove this rewrite for the
0.15 release and consider a reimplementation from scratch for the 1.0
release.
 

Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/1cbfdef3
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/1cbfdef3
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/1cbfdef3

Branch: refs/heads/master
Commit: 1cbfdef3fad5235b692c372ec94bcade7f1b9ff9
Parents: 30caf36
Author: Matthias Boehm 
Authored: Tue Sep 5 00:51:52 2017 -0700
Committer: Matthias Boehm 
Committed: Tue Sep 5 00:51:52 2017 -0700

--
 .../sysml/hops/rewrite/HopRewriteUtils.java |  18 +-
 .../controlprogram/parfor/opt/OptNode.java  |   5 +
 .../parfor/opt/OptimizerRuleBased.java  | 903 +--
 .../updateinplace/UpdateInPlaceTest.java|  46 -
 4 files changed, 52 insertions(+), 920 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/1cbfdef3/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
--
diff --git a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java 
b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
index 1bf381c..dcc740e 100644
--- a/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
+++ b/src/main/java/org/apache/sysml/hops/rewrite/HopRewriteUtils.java
@@ -31,6 +31,7 @@ import org.apache.sysml.hops.AggBinaryOp;
 import org.apache.sysml.hops.AggUnaryOp;
 import org.apache.sysml.hops.BinaryOp;
 import org.apache.sysml.hops.DataOp;
+import org.apache.sysml.hops.FunctionOp;
 import org.apache.sysml.hops.Hop;
 import org.apache.sysml.hops.Hop.AggOp;
 import org.apache.sysml.hops.Hop.DataGenMethod;
@@ -1040,11 +1041,18 @@ public class HopRewriteUtils
return ret;
}

-   public static boolean alwaysRequiresReblock(Hop hop)
-   {
-   return (hop instanceof DataOp 
-&& 
((DataOp)hop).getDataOpType()==DataOpTypes.PERSISTENTREAD
-&& 
((DataOp)hop).getInputFormatType()!=FileFormatTypes.BINARY);
+   public static boolean alwaysRequiresReblock(Hop hop) {
+   return (hop instanceof DataOp
+   && 
((DataOp)hop).getDataOpType()==DataOpTypes.PERSISTENTREAD
+&& 
((DataOp)hop).getInputFormatType()!=FileFormatTypes.BINARY);
+   }
+   
+   public static boolean containsFunctioOp(ArrayList candidates) {
+   if( candidates != null )
+   for( Hop cand : candidates )
+   if( cand instanceof FunctionOp )
+   return true;
+   return false;
}

public static boolean rHasSimpleReadChain(Hop root, String var)

http://git-wip-us.apache.org/repos/asf/systemml/blob/1cbfdef3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
--
diff --git 
a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java 
b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
index 193ce3e..2464bf6 100644
--- 
a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
+++ 
b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.Set;
 
+import org.apache.commons.lang.ArrayUtils;
 import org.apache.sysml.lops.Lop;
 import org.apache.sysml.runtime.controlprogram.ParForProgramBlock;
 import 
org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
@@ -120,6 +121,10 @@ public class OptNode
_ntype = type;
}

+   public boolean isNodeType(NodeType... types) {
+   return ArrayUtils.contains(types, _ntype);
+   }
+   
public ExecType getExecType() {
return _etype;
}