Repository: incubator-systemml Updated Branches: refs/heads/master 35f7d4140 -> 6bbae619a
[SYSTEMML-561] Performance dense matrix-frame casting (column append) This patch adds a special case for matrix-frame indexing with pure double schema (default) to avoid object creation, a logarithmic number of array re-allocations per column, and cache unfriendly behavior. For example, on a scenario of 1Mx1k matrix-frame conversion the results (single-threaded) were as follows: * original runtime: 48.2s * w/ array column append: 20.8s * w/ cache-conscious array column append: 16.1s Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/9e047199 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/9e047199 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/9e047199 Branch: refs/heads/master Commit: 9e0471999d5a97b285d52232c3dbb732f207ad8d Parents: 35f7d41 Author: Matthias Boehm <[email protected]> Authored: Mon Jun 6 20:50:16 2016 -0700 Committer: Matthias Boehm <[email protected]> Committed: Mon Jun 6 20:50:16 2016 -0700 ---------------------------------------------------------------------- .../sysml/runtime/matrix/data/FrameBlock.java | 15 +++++++- .../sysml/runtime/util/DataConverter.java | 36 +++++++++++++++++--- 2 files changed, 45 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9e047199/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java index d9cf8c3..268ba44 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java @@ -201,10 +201,19 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable private static List<String> createColNames(int size) { ArrayList<String> ret = new ArrayList<String>(size); for( int i=1; i<=size; i++ ) - ret.add("C"+i); + ret.add(createColName(i)); return ret; } + /** + * + * @param i + * @return + */ + private static String createColName(int i) { + return "C" + i; + } + /////// // basic get and set functionality @@ -282,6 +291,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable public void appendColumn(String[] col) { ensureColumnCompatibility(col.length); _schema.add(ValueType.STRING); + _colnames.add(createColName(_schema.size())); _coldata.add(new StringArray(col)); _numRows = col.length; } @@ -296,6 +306,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable public void appendColumn(boolean[] col) { ensureColumnCompatibility(col.length); _schema.add(ValueType.BOOLEAN); + _colnames.add(createColName(_schema.size())); _coldata.add(new BooleanArray(col)); _numRows = col.length; } @@ -310,6 +321,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable public void appendColumn(long[] col) { ensureColumnCompatibility(col.length); _schema.add(ValueType.INT); + _colnames.add(createColName(_schema.size())); _coldata.add(new LongArray(col)); _numRows = col.length; } @@ -324,6 +336,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable public void appendColumn(double[] col) { ensureColumnCompatibility(col.length); _schema.add(ValueType.DOUBLE); + _colnames.add(createColName(_schema.size())); _coldata.add(new DoubleArray(col)); _numRows = col.length; } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9e047199/src/main/java/org/apache/sysml/runtime/util/DataConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java index 6974d9b..eddd2e3 100644 --- a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java +++ b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java @@ -761,12 +761,38 @@ public class DataConverter } else //DENSE { - for( int i=0; i<mb.getNumRows(); i++ ) { - for( int j=0; j<mb.getNumColumns(); j++ ) { - row[j] = UtilFunctions.doubleToObject( - schema.get(j), mb.quickGetValue(i, j)); + int dFreq = Collections.frequency(schema, ValueType.DOUBLE); + + if( dFreq == schema.size() ) { + // special case double schema (without cell-object creation, + // col pre-allocation, and cache-friendly row-column copy) + int m = mb.getNumRows(); + int n = mb.getNumColumns(); + double[] a = mb.getDenseBlock(); + double[][] c = new double[n][m]; + int blocksizeIJ = 16; //blocks of a/c+overhead in L1 cache + if( !mb.isEmptyBlock(false) ) + for( int bi=0; bi<m; bi+=blocksizeIJ ) + for( int bj=0; bj<n; bj+=blocksizeIJ ) { + int bimin = Math.min(bi+blocksizeIJ, m); + int bjmin = Math.min(bj+blocksizeIJ, n); + for( int i=bi, aix=bi*n; i<bimin; i++, aix+=n ) + for( int j=bj; j<bjmin; j++ ) + c[j][i] = a[aix+j]; + } + frame.reset(); + for( int j=0; j<n; j++ ) + frame.appendColumn(c[j]); + } + else { + // general case + for( int i=0; i<mb.getNumRows(); i++ ) { + for( int j=0; j<mb.getNumColumns(); j++ ) { + row[j] = UtilFunctions.doubleToObject( + schema.get(j), mb.quickGetValue(i, j)); + } + frame.appendRow(row); } - frame.appendRow(row); } }
