[1/2] systemml git commit: [SYSTEMML-1743] Create lightweight SystemML artifact

2017-09-14 Thread deron 
Repository: systemml
Updated Branches:
  refs/heads/master 3c7a6eb70 -> 139ddfeed


http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/assembly/lite/LICENSE
--
diff --git a/src/assembly/lite/LICENSE b/src/assembly/lite/LICENSE
new file mode 100644
index 000..9c3793f
--- /dev/null
+++ b/src/assembly/lite/LICENSE
@@ -0,0 +1,293 @@
+ Apache License
+   Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+  "License" shall mean the terms and conditions for use, reproduction,
+  and distribution as defined by Sections 1 through 9 of this document.
+
+  "Licensor" shall mean the copyright owner or entity authorized by
+  the copyright owner that is granting the License.
+
+  "Legal Entity" shall mean the union of the acting entity and all
+  other entities that control, are controlled by, or are under common
+  control with that entity. For the purposes of this definition,
+  "control" means (i) the power, direct or indirect, to cause the
+  direction or management of such entity, whether by contract or
+  otherwise, or (ii) ownership of fifty percent (50%) or more of the
+  outstanding shares, or (iii) beneficial ownership of such entity.
+
+  "You" (or "Your") shall mean an individual or Legal Entity
+  exercising permissions granted by this License.
+
+  "Source" form shall mean the preferred form for making modifications,
+  including but not limited to software source code, documentation
+  source, and configuration files.
+
+  "Object" form shall mean any form resulting from mechanical
+  transformation or translation of a Source form, including but
+  not limited to compiled object code, generated documentation,
+  and conversions to other media types.
+
+  "Work" shall mean the work of authorship, whether in Source or
+  Object form, made available under the License, as indicated by a
+  copyright notice that is included in or attached to the work
+  (an example is provided in the Appendix below).
+
+  "Derivative Works" shall mean any work, whether in Source or Object
+  form, that is based on (or derived from) the Work and for which the
+  editorial revisions, annotations, elaborations, or other modifications
+  represent, as a whole, an original work of authorship. For the purposes
+  of this License, Derivative Works shall not include works that remain
+  separable from, or merely link (or bind by name) to the interfaces of,
+  the Work and Derivative Works thereof.
+
+  "Contribution" shall mean any work of authorship, including
+  the original version of the Work and any modifications or additions
+  to that Work or Derivative Works thereof, that is intentionally
+  submitted to Licensor for inclusion in the Work by the copyright owner
+  or by an individual or Legal Entity authorized to submit on behalf of
+  the copyright owner. For the purposes of this definition, "submitted"
+  means any form of electronic, verbal, or written communication sent
+  to the Licensor or its representatives, including but not limited to
+  communication on electronic mailing lists, source code control systems,
+  and issue tracking systems that are managed by, or on behalf of, the
+  Licensor for the purpose of discussing and improving the Work, but
+  excluding communication that is conspicuously marked or otherwise
+  designated in writing by the copyright owner as "Not a Contribution."
+
+  "Contributor" shall mean Licensor and any individual or Legal Entity
+  on behalf of whom a Contribution has been received by Licensor and
+  subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+  this License, each Contributor hereby grants to You a perpetual,
+  worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+  copyright license to reproduce, prepare Derivative Works of,
+  publicly display, publicly perform, sublicense, and distribute the
+  Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+  this License, each Contributor hereby grants to You a perpetual,
+  worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+  (except as stated in this section) patent license to make, have made,
+  use, offer to sell, sell, import, and otherwise transfer the Work,
+  where such license applies only to those patent claims licensable
+  by such Contributor that are necessarily infringed by their
+  Contribution(s) alone or by combination of their Contribution(s)
+  with

[2/2] systemml git commit: [SYSTEMML-1743] Create lightweight SystemML artifact

2017-09-14 Thread deron 
[SYSTEMML-1743] Create lightweight SystemML artifact

Create a lightweight SystemML jar file based on the execution of
a particular set of code, such as JMLC. In addition to generating
a minimal jar file that corresponds to this code, automatically
generate maven assembly dependency sets that can be used to build
a 'lite' jar that can be activated via a 'lite' maven profile.
This essentially creates an minimum-size uberjar that can perform
tasks such as scoring with JMLC, where distributed operations on
Spark and Hadoop are not desired.

Closes #578.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/139ddfee
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/139ddfee
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/139ddfee

Branch: refs/heads/master
Commit: 139ddfeed17b6a3d41fe91e9095253cf86dae090
Parents: 3c7a6eb
Author: Deron Eriksson 
Authored: Thu Sep 14 16:07:33 2017 -0700
Committer: Deron Eriksson 
Committed: Thu Sep 14 16:07:33 2017 -0700

--
 pom.xml |   29 +
 src/assembly/lite.xml   | 1359 ++
 src/assembly/lite/LICENSE   |  293 
 src/assembly/lite/NOTICE|5 +
 src/assembly/lite/log4j.properties  |   22 +
 src/assembly/lite/systemml-lite.txt |   18 +
 .../apache/sysml/conf/ConfigurationManager.java |   11 +
 .../parfor/stat/InfrastructureAnalyzer.java |   69 +-
 .../org/apache/sysml/utils/lite/BuildLite.java  |  822 +++
 .../sysml/utils/lite/BuildLiteExecution.java|  709 +
 .../org/apache/sysml/utils/lite/LiteCheck.java  |   49 +
 src/main/python/pre_setup.py|3 +-
 12 files changed, 3345 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/pom.xml
--
diff --git a/pom.xml b/pom.xml
index eb4337d..2fb8a48 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1163,6 +1163,35 @@

 

+   
+   lite
+   
+   
+   
+   
maven-assembly-plugin
+   
+   
gnu
+   
+   
+   
+   
create-lite-jar
+   
package
+   
+   
single
+   
+   
+   

+   
src/assembly/lite.xml
+   

+   
+   
+   
+   
+   
+   
+   
+
+   

standalone-jar

http://git-wip-us.apache.org/repos/asf/systemml/blob/139ddfee/src/assembly/lite.xml
--
diff --git a/src/assembly/lite.xml b/src/assembly/lite.xml
new file mode 100644
index 000..908c72e
--- /dev/null
+++ b/src/assembly/lite.xml
@@ -0,0 +1,1359 @@
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3;
+   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+   
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3
 http://maven.apache.org/xsd/assembly-1.1.3.xsd;>
+   
+   lite
+
+   
+   jar
+   
+
+   false
+
+   
+   
+   ${basedir}/src/assembly/lite
+   
+   LICENSE
+   NOTICE
+   systemml-lite.txt
+   
+   META-INF
+   
+
+   
+   ${basedir}/src/assembly/lite
+   
+   log4j.properties
+

systemml git commit: [SYSTEMML-1906] Additional tuning codegen row ops over compressed data

2017-09-14 Thread mboehm7 
Repository: systemml
Updated Branches:
  refs/heads/master 34c8a9231 -> 3c7a6eb70


[SYSTEMML-1906] Additional tuning codegen row ops over compressed data

This patch further improves upon the previous performance enhancements
for codegen row-wise operations over compressed matrices. In detail,
this includes, (1) improved memory efficiency due to fewer partial
buffers, (2) the elimination of redundant partial row copies, and (3)
the elimination of an unnecessary initialization scan of the code vector
per segment. 

Together these changes improved the performance over Airline78 and
Mnist8m from 995ms to 735m and from 22s to 15.2s, respectively.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/3c7a6eb7
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/3c7a6eb7
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/3c7a6eb7

Branch: refs/heads/master
Commit: 3c7a6eb70a160e338ce3204c052a6930de804c67
Parents: 34c8a92
Author: Matthias Boehm 
Authored: Thu Sep 14 15:20:25 2017 -0700
Committer: Matthias Boehm 
Committed: Thu Sep 14 15:21:01 2017 -0700

--
 .../apache/sysml/runtime/compress/ColGroup.java | 12 +++-
 .../sysml/runtime/compress/ColGroupDDC.java | 16 +--
 .../sysml/runtime/compress/ColGroupOLE.java | 30 +---
 .../sysml/runtime/compress/ColGroupRLE.java | 19 ++---
 .../runtime/compress/ColGroupUncompressed.java  | 17 +--
 .../runtime/compress/CompressedMatrixBlock.java | 23 ++-
 6 files changed, 56 insertions(+), 61 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/3c7a6eb7/src/main/java/org/apache/sysml/runtime/compress/ColGroup.java
--
diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroup.java 
b/src/main/java/org/apache/sysml/runtime/compress/ColGroup.java
index ba6509c..d6187e9 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/ColGroup.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroup.java
@@ -281,7 +281,7 @@ public abstract class ColGroup implements Serializable
 * @param ru row upper index, exclusive
 * @return an iterator instance
 */
-   public abstract Iterator getRowIterator(int rl, int ru);
+   public abstract ColGroupRowIterator getRowIterator(int rl, int ru);

/**
 * Count the number of non-zeros per row
@@ -291,4 +291,14 @@ public abstract class ColGroup implements Serializable
 * @param ru row upper bound, exclusive
 */
protected abstract void countNonZerosPerRow(int[] rnnz, int rl, int ru);
+
+   /**
+* Base class for column group row iterators. We do not
+* implement the default Iterator interface in order to
+* avoid unnecessary value copies per group.
+*/
+   protected abstract class ColGroupRowIterator {
+   public abstract boolean hasNext();
+   public abstract void next(double[] buff);
+   }
 }

http://git-wip-us.apache.org/repos/asf/systemml/blob/3c7a6eb7/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC.java
--
diff --git a/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC.java 
b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC.java
index 3618651..492f0b0 100644
--- a/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC.java
+++ b/src/main/java/org/apache/sysml/runtime/compress/ColGroupDDC.java
@@ -256,7 +256,7 @@ public abstract class ColGroupDDC extends ColGroupValue
}

@Override
-   public Iterator getRowIterator(int rl, int ru) {
+   public ColGroupRowIterator getRowIterator(int rl, int ru) {
return new DDCRowIterator(rl, ru);
}

@@ -305,12 +305,9 @@ public abstract class ColGroupDDC extends ColGroupValue
}
}

-   private class DDCRowIterator implements Iterator
+   private class DDCRowIterator extends ColGroupRowIterator
{
-   //iterator configuration 
private final int _ru;
-   //iterator state
-   private final double[] _buff = new double[getNumCols()]; 
private int _rpos = -1;

public DDCRowIterator(int rl, int ru) {
@@ -324,13 +321,14 @@ public abstract class ColGroupDDC extends ColGroupValue
}
 
@Override
-   public double[] next() {
-   //copy entire value tuple and 
+   public void next(double[] buff) {
+   //copy entire value

[systemml] Git Push Summary

2017-09-14 Thread acs_s 
Repository: systemml
Updated Tags:  refs/tags/v0.1.0-rc1 [deleted] 633a86b4f

systemml git commit: [MINOR] Fix build number

2017-09-14 Thread acs_s 
Repository: systemml
Updated Branches:
  refs/heads/master 03c304649 -> 34c8a9231


[MINOR] Fix build number


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/34c8a923
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/34c8a923
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/34c8a923

Branch: refs/heads/master
Commit: 34c8a923185020089e7acef75a3b97abee55e584
Parents: 03c3046
Author: Arvind Surve 
Authored: Thu Sep 14 14:41:09 2017 -0700
Committer: Arvind Surve 
Committed: Thu Sep 14 14:41:09 2017 -0700

--
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/34c8a923/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 88fb9fc..eb4337d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
18

org.apache.systemml
-   0.2.0-SNAPSHOT
+   1.0.0-SNAPSHOT
systemml
jar
SystemML

systemml git commit: [maven-release-plugin] prepare for next development iteration

2017-09-14 Thread acs_s 
Repository: systemml
Updated Branches:
  refs/heads/master 143fa7848 -> 03c304649


[maven-release-plugin] prepare for next development iteration


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/03c30464
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/03c30464
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/03c30464

Branch: refs/heads/master
Commit: 03c304649263905f00cd1ceaf63ef3cd4896a8db
Parents: 143fa78
Author: Arvind Surve 
Authored: Thu Sep 14 14:29:46 2017 -0700
Committer: Arvind Surve 
Committed: Thu Sep 14 14:29:46 2017 -0700

--
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/03c30464/pom.xml
--
diff --git a/pom.xml b/pom.xml
index 1843c36..88fb9fc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
18

org.apache.systemml
-   0.1.0
+   0.2.0-SNAPSHOT
systemml
jar
SystemML
@@ -41,7 +41,7 @@
scm:git:g...@github.com:apache/systemml

scm:git:https://git-wip-us.apache.org/repos/asf/systemml

https://git-wip-us.apache.org/repos/asf?p=systemml.git
-   v0.1.0-rc1
+   HEAD


JIRA

[systemml] Git Push Summary

2017-09-14 Thread acs_s 
Repository: systemml
Updated Tags:  refs/tags/v0.1.0-rc1 [created] 633a86b4f

systemml git commit: [maven-release-plugin] prepare release v0.1.0-rc1

2017-09-14 Thread acs_s 
Repository: systemml
Updated Branches:
  refs/heads/master c6d499d3e -> 143fa7848


[maven-release-plugin] prepare release v0.1.0-rc1


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/143fa784
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/143fa784
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/143fa784

Branch: refs/heads/master
Commit: 143fa7848e23d86c355f88b5756e3c4afee3cd12
Parents: c6d499d
Author: Arvind Surve 
Authored: Thu Sep 14 14:29:31 2017 -0700
Committer: Arvind Surve 
Committed: Thu Sep 14 14:29:31 2017 -0700

--
 pom.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/143fa784/pom.xml
--
diff --git a/pom.xml b/pom.xml
index eb4337d..1843c36 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
18

org.apache.systemml
-   1.0.0-SNAPSHOT
+   0.1.0
systemml
jar
SystemML
@@ -41,7 +41,7 @@
scm:git:g...@github.com:apache/systemml

scm:git:https://git-wip-us.apache.org/repos/asf/systemml

https://git-wip-us.apache.org/repos/asf?p=systemml.git
-   HEAD
+   v0.1.0-rc1


JIRA

systemml git commit: [MINOR] Refactored the locks to seperate out read and write lock

2017-09-14 Thread niketanpansare 
Repository: systemml
Updated Branches:
  refs/heads/master 0a984a43b -> c6d499d3e


[MINOR] Refactored the locks to seperate out read and write lock

- Refactoring the locks will avoid future bugs where the developer tries
  to obtain 2 write lock or a read lock on a write-locked objects, etc.
- I have also added a debugging utility to track potential memory leaks.

Closes #664.


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/c6d499d3
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/c6d499d3
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/c6d499d3

Branch: refs/heads/master
Commit: c6d499d3e27a1842ccf5987ab84c92eee72aa5c2
Parents: 0a984a4
Author: Niketan Pansare 
Authored: Thu Sep 14 13:20:46 2017 -0700
Committer: Niketan Pansare 
Committed: Thu Sep 14 13:23:14 2017 -0700

--
 .../context/ExecutionContext.java   |  2 +-
 .../instructions/gpu/GPUInstruction.java| 12 +++
 .../instructions/gpu/context/GPUContext.java| 52 ++--
 .../instructions/gpu/context/GPUObject.java | 84 ++--
 4 files changed, 117 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/c6d499d3/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
--
diff --git 
a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
 
b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
index b74c0dd..3b2436e 100644
--- 
a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
+++ 
b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java
@@ -374,7 +374,7 @@ public class ExecutionContext {
}
// The lock is added here for an output block
// so that any block currently in use is not deallocated by 
eviction on the GPU
-   mo.getGPUObject(getGPUContext(0)).addLock();
+   mo.getGPUObject(getGPUContext(0)).addWriteLock();
return mo;
}
 

http://git-wip-us.apache.org/repos/asf/systemml/blob/c6d499d3/src/main/java/org/apache/sysml/runtime/instructions/gpu/GPUInstruction.java
--
diff --git 
a/src/main/java/org/apache/sysml/runtime/instructions/gpu/GPUInstruction.java 
b/src/main/java/org/apache/sysml/runtime/instructions/gpu/GPUInstruction.java
index 9a6a3bb..2aa73b4 100644
--- 
a/src/main/java/org/apache/sysml/runtime/instructions/gpu/GPUInstruction.java
+++ 
b/src/main/java/org/apache/sysml/runtime/instructions/gpu/GPUInstruction.java
@@ -19,6 +19,8 @@
 
 package org.apache.sysml.runtime.instructions.gpu;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.sysml.api.DMLScript;
 import org.apache.sysml.lops.runtime.RunMRJobs;
 import org.apache.sysml.runtime.DMLRuntimeException;
@@ -26,6 +28,7 @@ import 
org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
 import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
 import org.apache.sysml.runtime.instructions.GPUInstructionParser;
 import org.apache.sysml.runtime.instructions.Instruction;
+import org.apache.sysml.runtime.instructions.gpu.context.GPUContext;
 import org.apache.sysml.runtime.matrix.data.Pair;
 import org.apache.sysml.runtime.matrix.operators.Operator;
 import org.apache.sysml.utils.GPUStatistics;
@@ -46,6 +49,8 @@ public abstract class GPUInstruction extends Instruction {
Builtin,
MatrixIndexing
};
+   
+   private static final Log LOG = 
LogFactory.getLog(GPUInstruction.class.getName());
 
// Memory/conversions
public final static String MISC_TIMER_HOST_TO_DEVICE =  "H2D";  
// time spent in bringing data to gpu (from host)
@@ -191,6 +196,13 @@ public abstract class GPUInstruction extends Instruction {
if(DMLScript.SYNCHRONIZE_GPU) {
jcuda.runtime.JCuda.cudaDeviceSynchronize();
}
+   if(LOG.isDebugEnabled()) {
+   for(GPUContext gpuCtx : ec.getGPUContexts()) {
+   if(gpuCtx != null)
+   gpuCtx.printMemoryInfo(getOpcode());
+   }
+   }
+   
}
 
/**

http://git-wip-us.apache.org/repos/asf/systemml/blob/c6d499d3/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUContext.java
--
diff --git

systemml git commit: [SYSTEMML-1906] Performance codegen row ops over compressed matrices

2017-09-14 Thread mboehm7 
Repository: systemml
Updated Branches:
  refs/heads/master 4cf95c92e -> 0a984a43b


[SYSTEMML-1906] Performance codegen row ops over compressed matrices

This patch improves the performance of codegen row-wise operations over
compressed dense and sparse matrices. So far we used dense/sparse row
iterators over the compressed matrix block, which internally reused the
existing column group iterators. However, for the OLE and RLE encoding
schemes, these iterators were realized via a set of value iterators,
which did not perform very well. The key challenge is the translation
from the value-based, column-wise compressed form to uncompressed rows. 

We now use dedicated row iterators for the individual column encoding
schemes. For DDC and UC, these are trivial. For OLE and RLE, however, we
determine a vector of dictionary codes per logical segment (via a single
pass over all value offset lists) and simply read out the value tuples
for these codes on the individual next calls. Furthermore, we now use
static task partitioning (with segment alignment) for multi-threaded row
operations to avoid unnecessary iterator initialization overhead. On the
Airline78 (dense) and Mnist8m (sparse) dataset, this patch improves
performance from 4.1s to 1s, and from 66s to 21s, respectively


Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/0a984a43
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/0a984a43
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/0a984a43

Branch: refs/heads/master
Commit: 0a984a43b1062aaf58bf7a2ce019adef264ba459
Parents: 4cf95c9
Author: Matthias Boehm 
Authored: Thu Sep 14 00:26:45 2017 -0700
Committer: Matthias Boehm 
Committed: Thu Sep 14 00:27:04 2017 -0700

--
 .../sysml/runtime/codegen/SpoofRowwise.java |  33 +--
 .../apache/sysml/runtime/compress/ColGroup.java |  19 ++
 .../sysml/runtime/compress/ColGroupDDC.java |  47 +++-
 .../sysml/runtime/compress/ColGroupDDC1.java|  12 +-
 .../sysml/runtime/compress/ColGroupDDC2.java|  14 +-
 .../sysml/runtime/compress/ColGroupOLE.java |  67 -
 .../sysml/runtime/compress/ColGroupRLE.java |  78 +-
 .../runtime/compress/ColGroupUncompressed.java  |  51 
 .../runtime/compress/CompressedMatrixBlock.java |  76 ++
 .../CompressedRowAggregateLargeTest.java| 270 +++
 .../functions/codegen/ZPackageSuite.java|   1 +
 11 files changed, 589 insertions(+), 79 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/systemml/blob/0a984a43/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java
--
diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java 
b/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java
index f1cef34..659059e 100644
--- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java
+++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofRowwise.java
@@ -29,6 +29,7 @@ import java.util.concurrent.Future;
 import java.util.stream.IntStream;
 
 import org.apache.sysml.runtime.DMLRuntimeException;
+import org.apache.sysml.runtime.compress.BitmapEncoder;
 import org.apache.sysml.runtime.compress.CompressedMatrixBlock;
 import org.apache.sysml.runtime.instructions.cp.DoubleObject;
 import org.apache.sysml.runtime.instructions.cp.ScalarObject;
@@ -185,20 +186,25 @@ public abstract class SpoofRowwise extends SpoofOperator
&& LibSpoofPrimitives.isFlipOuter(out.getNumRows(), 
out.getNumColumns());

//input preparation
+   MatrixBlock a = inputs.get(0);
SideInput[] b = prepInputMatrices(inputs, 1, inputs.size()-1, 
true, _tB1);
double[] scalars = prepInputScalars(scalarObjects);

//core parallel execute
ExecutorService pool = Executors.newFixedThreadPool( k );
-   int nk = UtilFunctions.roundToNext(Math.min(8*k,m/32), k);
+   int nk = (a instanceof CompressedMatrixBlock) ? k :
+   UtilFunctions.roundToNext(Math.min(8*k,m/32), k);
int blklen = (int)(Math.ceil((double)m/nk));
+   if( a instanceof CompressedMatrixBlock )
+   blklen = BitmapEncoder.getAlignedBlocksize(blklen);
+   
try
{
if( _type.isColumnAgg() || _type == RowType.FULL_AGG ) {
//execute tasks
ArrayList tasks = new 
ArrayList();
for( int i=0; i> taskret = 
pool.invokeAll(tasks); 
//aggregate partial results