Repository: incubator-systemml Updated Branches: refs/heads/master 28fe4fe8f -> bcf96e1cc
[SYSTEMML-1395] Robustness of parfor spark partitioning w/ unknown nnz Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/bcf96e1c Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/bcf96e1c Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/bcf96e1c Branch: refs/heads/master Commit: bcf96e1ccc32948fa2420bd5d9649d15a8d4d521 Parents: 28fe4fe Author: Matthias Boehm <[email protected]> Authored: Sat Mar 11 23:08:48 2017 -0800 Committer: Matthias Boehm <[email protected]> Committed: Sat Mar 11 23:08:48 2017 -0800 ---------------------------------------------------------------------- .../sysml/runtime/controlprogram/ParForProgramBlock.java | 10 ++++++---- .../runtime/controlprogram/caching/MatrixObject.java | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/bcf96e1c/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java index 3d1e6ee..3339dca 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java @@ -1068,11 +1068,13 @@ public class ParForProgramBlock extends ForProgramBlock //write matrices to HDFS, except DP matrix which is the input to the RemoteDPParForSpark job exportMatricesToHDFS(ec, _colocatedDPMatrix); - + // Step 4) submit MR job (wait for finished work) - OutputInfo inputOI = ((inputMatrix.getSparsity()<0.1 && inputDPF==PDataPartitionFormat.COLUMN_WISE)|| - (inputMatrix.getSparsity()<0.001 && inputDPF==PDataPartitionFormat.ROW_WISE))? - OutputInfo.BinaryCellOutputInfo : OutputInfo.BinaryBlockOutputInfo; + //TODO runtime support for binary cell partitioning + //OutputInfo inputOI = ((inputMatrix.getSparsity()<0.1 && inputDPF==PDataPartitionFormat.COLUMN_WISE)|| + // (inputMatrix.getSparsity()<0.001 && inputDPF==PDataPartitionFormat.ROW_WISE))? + // OutputInfo.BinaryCellOutputInfo : OutputInfo.BinaryBlockOutputInfo; + OutputInfo inputOI = OutputInfo.BinaryBlockOutputInfo; RemoteParForJobReturn ret = RemoteDPParForSpark.runJob(_ID, itervar.getName(), _colocatedDPMatrix, program, clsMap, resultFile, inputMatrix, ec, inputDPF, inputOI, _tSparseCol, _enableCPCaching, _numThreads ); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/bcf96e1c/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java index b73adb0..48855ee 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java @@ -185,7 +185,7 @@ public class MatrixObject extends CacheableData<MatrixBlock> public double getSparsity() { MatrixCharacteristics mc = getMatrixCharacteristics(); - return ((double)mc.getNonZeros())/mc.getRows()/mc.getCols(); + return OptimizerUtils.getSparsity(mc); } // *********************************************
