Repository: systemml Updated Branches: refs/heads/master cf200e1dd -> d596d2214
[SYSTEMML-2047] Parallel array allocation for large dense blocks This patch modifies the new large dense block to allocate and initialize its underlying arrays in a multi-threaded manner. On a scenario of reading a 4M x 1k (32GB) binary block matrix from HDFS, this patch improved the end-to-end read time from 23.1s to 20.8s. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d596d221 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d596d221 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d596d221 Branch: refs/heads/master Commit: d596d2214b6e745ff61ab41178b5e5ff8cfacb54 Parents: cf200e1 Author: Matthias Boehm <[email protected]> Authored: Wed Jan 3 21:57:26 2018 -0800 Committer: Matthias Boehm <[email protected]> Committed: Wed Jan 3 21:57:26 2018 -0800 ---------------------------------------------------------------------- .../runtime/matrix/data/DenseBlockLDRB.java | 23 +++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/d596d221/src/main/java/org/apache/sysml/runtime/matrix/data/DenseBlockLDRB.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/DenseBlockLDRB.java b/src/main/java/org/apache/sysml/runtime/matrix/data/DenseBlockLDRB.java index b356b6f..2011629 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/DenseBlockLDRB.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/DenseBlockLDRB.java @@ -21,6 +21,7 @@ package org.apache.sysml.runtime.matrix.data; import java.util.Arrays; +import java.util.stream.IntStream; import org.apache.sysml.runtime.util.UtilFunctions; @@ -28,6 +29,8 @@ public class DenseBlockLDRB extends DenseBlock { private static final long serialVersionUID = -7285459683402612969L; + private static final boolean PARALLEL_ALLOC = true; + private double[][] data; private int rlen; private int clen; @@ -56,6 +59,7 @@ public class DenseBlockLDRB extends DenseBlock reset(rlen, clen, blen, v); } + @SuppressWarnings("resource") private void reset(int rlen, int clen, int blen, double v) { long llen = (long) rlen * clen; int numPart = (int)Math.ceil((double)rlen / blen); @@ -67,17 +71,24 @@ public class DenseBlockLDRB extends DenseBlock } else { data = new double[numPart][]; - for(int i=0; i<numPart; i++) { - int lrlen = (int)(Math.min((i+1)*blen,rlen)-i*blen); - data[i] = new double[lrlen*clen]; - if( v != 0 ) - Arrays.fill(data[i], v); - } + IntStream range = PARALLEL_ALLOC ? + IntStream.range(0, numPart).parallel() : + IntStream.range(0, numPart); + range.forEach(i -> + data[i] = allocArray(i, rlen, clen, blen, v)); } this.rlen = rlen; this.clen = clen; this.blen = blen; } + + private static double[] allocArray(int i, int rlen, int clen, int blen, double v) { + int lrlen = (int)(Math.min((i+1)*blen,rlen)-i*blen); + double[] ret = new double[lrlen*clen]; + if( v != 0 ) + Arrays.fill(ret, v); + return ret; + } @Override public int numRows() {
