Repository: incubator-systemml Updated Branches: refs/heads/master 0c85c1e52 -> 7e2383bfa
[MINOR] Addressed corner cases (i.e. empty blocks and extremely large blocks) as well added recomputing nnz in the converters Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7e2383bf Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7e2383bf Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7e2383bf Branch: refs/heads/master Commit: 7e2383bfa51fc51cfe637680f94758eb1ae193cf Parents: 0c85c1e Author: Niketan Pansare <[email protected]> Authored: Thu Feb 16 21:41:07 2017 -0800 Committer: Niketan Pansare <[email protected]> Committed: Thu Feb 16 21:42:05 2017 -0800 ---------------------------------------------------------------------- .../spark/utils/RDDConverterUtilsExt.java | 34 ++++++++++++++------ 1 file changed, 25 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7e2383bf/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java index 973db64..dea5601 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java @@ -154,8 +154,10 @@ public class RDDConverterUtilsExt double val = buf1.getDouble(); int rowIndex = buf2.getInt(); int colIndex = buf3.getInt(); - mb.setValue(rowIndex, colIndex, val); // TODO: Improve the performance + mb.setValue(rowIndex, colIndex, val); } + mb.recomputeNonZeros(); + mb.examSparsity(); return mb; } @@ -169,7 +171,10 @@ public class RDDConverterUtilsExt throw new DMLRuntimeException("Convertion to sparse format not supported"); } else { - double [] denseBlock = new double[rlen*clen]; + long limit = rlen*clen; + if( limit > Integer.MAX_VALUE ) + throw new DMLRuntimeException("Dense NumPy array of size " + limit + " cannot be converted to MatrixBlock"); + double [] denseBlock = new double[(int) limit]; ByteBuffer buf = ByteBuffer.wrap(data); buf.order(ByteOrder.nativeOrder()); for(int i = 0; i < rlen*clen; i++) { @@ -177,6 +182,7 @@ public class RDDConverterUtilsExt } mb.init( denseBlock, rlen, clen ); } + mb.recomputeNonZeros(); mb.examSparsity(); return mb; } @@ -185,17 +191,27 @@ public class RDDConverterUtilsExt byte [] ret = null; if(mb.isInSparseFormat()) { mb.sparseToDense(); -// throw new DMLRuntimeException("Sparse to dense conversion is not yet implemented"); } + long limit = mb.getNumRows()*mb.getNumColumns(); + int times = Double.SIZE / Byte.SIZE; + if( limit * times > Integer.MAX_VALUE ) + throw new DMLRuntimeException("MatrixBlock of size " + limit + " cannot be converted to dense numpy array"); + ret = new byte[(int) (limit * times)]; + double [] denseBlock = mb.getDenseBlock(); - if(denseBlock == null) { - throw new DMLRuntimeException("Sparse to dense conversion is not yet implemented"); + if(mb.isEmptyBlock()) { + for(int i=0;i < limit;i++){ + ByteBuffer.wrap(ret, i*times, times).order(ByteOrder.nativeOrder()).putDouble(0); + } } - int times = Double.SIZE / Byte.SIZE; - ret = new byte[denseBlock.length * times]; - for(int i=0;i < denseBlock.length;i++){ - ByteBuffer.wrap(ret, i*times, times).order(ByteOrder.nativeOrder()).putDouble(denseBlock[i]); + else if(denseBlock == null) { + throw new DMLRuntimeException("Error while dealing with empty blocks."); + } + else { + for(int i=0;i < denseBlock.length;i++){ + ByteBuffer.wrap(ret, i*times, times).order(ByteOrder.nativeOrder()).putDouble(denseBlock[i]); + } } return ret;
