Repository: systemml Updated Branches: refs/heads/master 3cda8a981 -> 9a6a9bd76
[SYSTEMML-2023] Fix RDD collect of compressed distributed matrices This patch fixes the RDD collect code path for robustness regarding compressed matrix blocks. So far collecting compressed matrices fails with null pointer exceptions due to missing dense or sparse blocks. Since the target matrix is allocated in uncompressed form before individual blocks are accessed, we now decompress blocks if necessary for robustness. In the future, we should keep the meta data in matrix object handles and preserve the compressed representation. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/079b4936 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/079b4936 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/079b4936 Branch: refs/heads/master Commit: 079b4936ee6b601389e7c24eaa43fcbb56213acc Parents: 3cda8a9 Author: Matthias Boehm <[email protected]> Authored: Sun Nov 19 22:48:28 2017 -0800 Committer: Matthias Boehm <[email protected]> Committed: Sun Nov 19 22:48:28 2017 -0800 ---------------------------------------------------------------------- .../controlprogram/context/SparkExecutionContext.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/079b4936/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java index 39e52f5..84fdd88 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java @@ -46,6 +46,7 @@ import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.lops.Checkpoint; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.DMLRuntimeException; +import org.apache.sysml.runtime.compress.CompressedMatrixBlock; import org.apache.sysml.runtime.controlprogram.Program; import org.apache.sysml.runtime.controlprogram.caching.CacheableData; import org.apache.sysml.runtime.controlprogram.caching.FrameObject; @@ -832,13 +833,17 @@ public class SparkExecutionContext extends ExecutionContext //unpack index-block pair MatrixIndexes ix = keyval._1(); MatrixBlock block = keyval._2(); - + //compute row/column block offsets int row_offset = (int)(ix.getRowIndex()-1)*brlen; int col_offset = (int)(ix.getColumnIndex()-1)*bclen; int rows = block.getNumRows(); int cols = block.getNumColumns(); - + + //handle compressed blocks (decompress for robustness) + if( block instanceof CompressedMatrixBlock ) + block = ((CompressedMatrixBlock)block).decompress(); + //append block if( sparse ) { //SPARSE OUTPUT //append block to sparse target in order to avoid shifting, where
