Repository: systemml Updated Branches: refs/heads/master f700df2c8 -> 68a7b44b5
[HOTFIX][SYSTEMML-2235] Fix bufferpool serialization compressed blocks The recent fix of a bufferpool leak in SYSTEMML-2234 also made a small improvement to the buffer pool threshold handling, which revealed a hidden bug of missing bufferpool serialization of compressed matrix blocks. This patch fixes this issue by correctly handling the shallow serialize meta data, implementing missing char serialization functionality, and fixing the reported exact size of compressed blocks in serialized form (wrong header size). Furthermore, this patch also makes some refactoring to eliminate redundancy in the cache data and buffered data streams. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/68a7b44b Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/68a7b44b Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/68a7b44b Branch: refs/heads/master Commit: 68a7b44b5c6e783dd9d5113d3ee03b1b1a019afa Parents: f700df2 Author: Matthias Boehm <mboe...@gmail.com> Authored: Fri Apr 6 14:53:13 2018 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Fri Apr 6 15:06:16 2018 -0700 ---------------------------------------------------------------------- .../runtime/compress/CompressedMatrixBlock.java | 28 ++- .../controlprogram/caching/CacheDataInput.java | 53 ++---- .../controlprogram/caching/CacheDataOutput.java | 65 +++---- .../sysml/runtime/io/IOUtilFunctions.java | 52 ++++++ .../util/FastBufferedDataInputStream.java | 82 +++------ .../util/FastBufferedDataOutputStream.java | 171 +++++++------------ .../codegen/CompressedOuterProductTest.java | 8 +- 7 files changed, 204 insertions(+), 255 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java index 569ef14..2051f34 100644 --- a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java @@ -751,7 +751,7 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable timePhase4 = t4; timePhase5 = t5; } - } + } @Override public double quickGetValue(int r, int c) { @@ -768,26 +768,38 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable //find row value return grp.get(r, c); - } + } ////////////////////////////////////////// // Serialization / Deserialization @Override - public long getExactSizeOnDisk() - { + public long getExactSizeOnDisk() { //header information - long ret = 12; - + long ret = 22; for( ColGroup grp : _colGroups ) { ret += 1; //type info ret += grp.getExactSizeOnDisk(); } - return ret; } @Override + public boolean isShallowSerialize() { + return false; + } + + @Override + public boolean isShallowSerialize(boolean inclConvert) { + return false; + } + + @Override + public void toShallowSerializeBlock() { + //do nothing + } + + @Override public void readFields(DataInput in) throws IOException { @@ -980,7 +992,7 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable ret2 = (CompressedMatrixBlock) ret; ret2.reset(m, n); } - + //shallow copy of lhs column groups ret2.allocateColGroupList(); ret2._colGroups.addAll(_colGroups); http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java index 8f16ae1..dde2211 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataInput.java @@ -22,18 +22,17 @@ package org.apache.sysml.runtime.controlprogram.caching; import java.io.DataInput; import java.io.IOException; +import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.matrix.data.MatrixBlockDataInput; import org.apache.sysml.runtime.matrix.data.SparseBlock; public class CacheDataInput implements DataInput, MatrixBlockDataInput { - protected byte[] _buff; protected int _bufflen; protected int _count; - public CacheDataInput( byte[] mem ) - { + public CacheDataInput( byte[] mem ) { _buff = mem; _bufflen = _buff.length; _count = 0; @@ -55,17 +54,13 @@ public class CacheDataInput implements DataInput, MatrixBlockDataInput } @Override - public boolean readBoolean() - throws IOException - { + public boolean readBoolean() throws IOException { //mask to adhere to the input stream semantic return ( (_buff[_count++] & 0xFF) != 0 ); } @Override - public byte readByte() - throws IOException - { + public byte readByte() throws IOException { //mask to adhere to the input stream semantic return (byte) (_buff[_count++] & 0xFF); } @@ -87,26 +82,22 @@ public class CacheDataInput implements DataInput, MatrixBlockDataInput @Override public char readChar() throws IOException { - throw new IOException("Not supported."); + int ret = baToShort(_buff, _count); + _count += 2; + return (char) ret; } @Override - public int readInt() - throws IOException - { + public int readInt() throws IOException { int ret = baToInt(_buff, _count); _count += 4; - return ret; } @Override - public long readLong() - throws IOException - { + public long readLong() throws IOException { long ret = baToLong(_buff, _count); _count += 8; - return ret; } @@ -200,25 +191,15 @@ public class CacheDataInput implements DataInput, MatrixBlockDataInput return nnz; } - private static int baToInt( byte[] ba, final int off ) - { - //shift and add 4 bytes into single int - return ((ba[off+0] & 0xFF) << 24) + - ((ba[off+1] & 0xFF) << 16) + - ((ba[off+2] & 0xFF) << 8) + - ((ba[off+3] & 0xFF) << 0); + private static int baToShort( byte[] ba, final int off ) { + return IOUtilFunctions.baToShort(ba, off); } - private static long baToLong( byte[] ba, final int off ) - { - //shift and add 8 bytes into single long - return ((long)(ba[off+0] & 0xFF) << 56) + - ((long)(ba[off+1] & 0xFF) << 48) + - ((long)(ba[off+2] & 0xFF) << 40) + - ((long)(ba[off+3] & 0xFF) << 32) + - ((long)(ba[off+4] & 0xFF) << 24) + - ((long)(ba[off+5] & 0xFF) << 16) + - ((long)(ba[off+6] & 0xFF) << 8) + - ((long)(ba[off+7] & 0xFF) << 0); + private static int baToInt( byte[] ba, final int off ) { + return IOUtilFunctions.baToInt(ba, off); + } + + private static long baToLong( byte[] ba, final int off ) { + return IOUtilFunctions.baToLong(ba, off); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java index b9ed0a8..1b740d2 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheDataOutput.java @@ -37,7 +37,7 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput protected int _bufflen; protected int _count; - public CacheDataOutput( byte[] mem ) { + public CacheDataOutput( byte[] mem ) { _buff = mem; _bufflen = _buff.length; _count = 0; @@ -46,26 +46,25 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput @Override public void write(int b) throws IOException { _buff[_count++] = (byte)b; - } + } - @Override + @Override public void write(byte[] b) throws IOException { System.arraycopy(b, 0, _buff, _count, b.length); _count += b.length; } - - @Override + + @Override public void write(byte[] b, int off, int len) throws IOException { System.arraycopy(b, off, _buff, _count, len); _count += len; - } + } @Override public void writeBoolean(boolean v) throws IOException { _buff[_count++] = (byte)( v ? 1 : 0 ); } - @Override public void writeInt(int v) throws IOException { intToBa(v, _buff, _count); @@ -74,14 +73,14 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput @Override public void writeDouble(double v) throws IOException { - long tmp = Double.doubleToRawLongBits(v); + long tmp = Double.doubleToRawLongBits(v); longToBa(tmp, _buff, _count); _count += 8; } @Override public void writeByte(int v) throws IOException { - _buff[_count++] = (byte) v; + _buff[_count++] = (byte) v; } @Override @@ -91,7 +90,7 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput @Override public void writeChar(int v) throws IOException { - throw new IOException("Not supported."); + writeShort(v); } @Override @@ -106,7 +105,7 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput @Override public void writeLong(long v) throws IOException { - longToBa(v, _buff, _count); + longToBa(v, _buff, _count); _count += 8; } @@ -143,10 +142,10 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput } } - /////////////////////////////////////////////// - // Implementation of MatrixBlockDSMDataOutput - /////////////////////////////////////////////// - + /////////////////////////////////////////////// + // Implementation of MatrixBlockDSMDataOutput + /////////////////////////////////////////////// + @Override public void writeDoubleArray(int len, double[] varr) throws IOException @@ -155,10 +154,9 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput int off = _count; //serialize entire array into buffer - for( int i=0; i<len; i++ ) - { - long tmp = Double.doubleToRawLongBits(varr[i]); - longToBa(tmp, _buff, off+i*8); + for( int i=0; i<len; i++ ) { + long tmp = Double.doubleToRawLongBits(varr[i]); + longToBa(tmp, _buff, off+i*8); } //update buffer offset @@ -200,32 +198,15 @@ public class CacheDataOutput implements DataOutput, MatrixBlockDataOutput writeInt( 0 ); } - private static void shortToBa( final int val, byte[] ba, final int off ) - { - //shift and mask out 2 bytes - ba[ off+0 ] = (byte)((val >>> 8) & 0xFF); - ba[ off+1 ] = (byte)((val >>> 0) & 0xFF); + private static void shortToBa( final int val, byte[] ba, final int off ) { + IOUtilFunctions.shortToBa(val, ba, off); } - private static void intToBa( final int val, byte[] ba, final int off ) - { - //shift and mask out 4 bytes - ba[ off+0 ] = (byte)((val >>> 24) & 0xFF); - ba[ off+1 ] = (byte)((val >>> 16) & 0xFF); - ba[ off+2 ] = (byte)((val >>> 8) & 0xFF); - ba[ off+3 ] = (byte)((val >>> 0) & 0xFF); + private static void intToBa( final int val, byte[] ba, final int off ) { + IOUtilFunctions.intToBa(val, ba, off); } - private static void longToBa( final long val, byte[] ba, final int off ) - { - //shift and mask out 8 bytes - ba[ off+0 ] = (byte)((val >>> 56) & 0xFF); - ba[ off+1 ] = (byte)((val >>> 48) & 0xFF); - ba[ off+2 ] = (byte)((val >>> 40) & 0xFF); - ba[ off+3 ] = (byte)((val >>> 32) & 0xFF); - ba[ off+4 ] = (byte)((val >>> 24) & 0xFF); - ba[ off+5 ] = (byte)((val >>> 16) & 0xFF); - ba[ off+6 ] = (byte)((val >>> 8) & 0xFF); - ba[ off+7 ] = (byte)((val >>> 0) & 0xFF); + private static void longToBa( final long val, byte[] ba, final int off ) { + IOUtilFunctions.longToBa(val, ba, off); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java index 4d388c9..b484dfc 100644 --- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java @@ -513,4 +513,56 @@ public class IOUtilFunctions fs.delete(fnameMtdCrc, false); } } + + public static int baToShort( byte[] ba, final int off ) { + //shift and add 2 bytes into single int + return ((ba[off+0] & 0xFF) << 8) + + ((ba[off+1] & 0xFF) << 0); + } + + public static int baToInt( byte[] ba, final int off ) { + //shift and add 4 bytes into single int + return ((ba[off+0] & 0xFF) << 24) + + ((ba[off+1] & 0xFF) << 16) + + ((ba[off+2] & 0xFF) << 8) + + ((ba[off+3] & 0xFF) << 0); + } + + public static long baToLong( byte[] ba, final int off ) { + //shift and add 8 bytes into single long + return ((long)(ba[off+0] & 0xFF) << 56) + + ((long)(ba[off+1] & 0xFF) << 48) + + ((long)(ba[off+2] & 0xFF) << 40) + + ((long)(ba[off+3] & 0xFF) << 32) + + ((long)(ba[off+4] & 0xFF) << 24) + + ((long)(ba[off+5] & 0xFF) << 16) + + ((long)(ba[off+6] & 0xFF) << 8) + + ((long)(ba[off+7] & 0xFF) << 0); + } + + public static void shortToBa( final int val, byte[] ba, final int off ) { + //shift and mask out 2 bytes + ba[ off+0 ] = (byte)((val >>> 8) & 0xFF); + ba[ off+1 ] = (byte)((val >>> 0) & 0xFF); + } + + public static void intToBa( final int val, byte[] ba, final int off ) { + //shift and mask out 4 bytes + ba[ off+0 ] = (byte)((val >>> 24) & 0xFF); + ba[ off+1 ] = (byte)((val >>> 16) & 0xFF); + ba[ off+2 ] = (byte)((val >>> 8) & 0xFF); + ba[ off+3 ] = (byte)((val >>> 0) & 0xFF); + } + + public static void longToBa( final long val, byte[] ba, final int off ) { + //shift and mask out 8 bytes + ba[ off+0 ] = (byte)((val >>> 56) & 0xFF); + ba[ off+1 ] = (byte)((val >>> 48) & 0xFF); + ba[ off+2 ] = (byte)((val >>> 40) & 0xFF); + ba[ off+3 ] = (byte)((val >>> 32) & 0xFF); + ba[ off+4 ] = (byte)((val >>> 24) & 0xFF); + ba[ off+5 ] = (byte)((val >>> 16) & 0xFF); + ba[ off+6 ] = (byte)((val >>> 8) & 0xFF); + ba[ off+7 ] = (byte)((val >>> 0) & 0xFF); + } } http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java index 197467d..9069859 100644 --- a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java +++ b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataInputStream.java @@ -25,13 +25,13 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; +import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.matrix.data.MatrixBlockDataInput; import org.apache.sysml.runtime.matrix.data.SparseBlock; import org.apache.sysml.runtime.matrix.data.SparseBlockCSR; public class FastBufferedDataInputStream extends FilterInputStream implements DataInput, MatrixBlockDataInput { - protected byte[] _buff; protected int _bufflen; @@ -41,9 +41,8 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da public FastBufferedDataInputStream( InputStream in, int size ) { super(in); - if (size <= 0) - throw new IllegalArgumentException("Buffer size <= 0"); + throw new IllegalArgumentException("Buffer size <= 0"); _buff = new byte[ size ]; _bufflen = size; } @@ -52,26 +51,21 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da // DataInput Implementation ///////////////////////////// - @Override - public void readFully(byte[] b) - throws IOException - { + public void readFully(byte[] b) throws IOException { readFully(b, 0, b.length); } @Override - public void readFully(byte[] b, int off, int len) - throws IOException - { + public void readFully(byte[] b, int off, int len) throws IOException { if (len < 0) - throw new IndexOutOfBoundsException(); + throw new IndexOutOfBoundsException(); int n = 0; while (n < len) { - int count = in.read(b, off + n, len - n); - if (count < 0) - throw new EOFException(); - n += count; + int count = in.read(b, off + n, len - n); + if (count < 0) + throw new EOFException(); + n += count; } } @@ -81,16 +75,12 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da } @Override - public boolean readBoolean() - throws IOException - { - return ( in.read() != 0 ); + public boolean readBoolean() throws IOException { + return in.read() != 0; } @Override - public byte readByte() - throws IOException - { + public byte readByte() throws IOException { return (byte)in.read(); } @@ -111,24 +101,19 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da @Override public char readChar() throws IOException { - throw new IOException("Not supported."); + readFully(_buff, 0, 2); + return (char)baToShort(_buff, 0); } @Override - public int readInt() - throws IOException - { + public int readInt() throws IOException { readFully(_buff, 0, 4); - return baToInt(_buff, 0); } @Override - public long readLong() - throws IOException - { + public long readLong() throws IOException { readFully(_buff, 0, 8); - return baToLong(_buff, 0); } @@ -138,11 +123,8 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da } @Override - public double readDouble() - throws IOException - { + public double readDouble() throws IOException { readFully(_buff, 0, 8); - long tmp = baToLong(_buff, 0); return Double.longBitsToDouble( tmp ); } @@ -160,7 +142,7 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da /////////////////////////////////////////////// // Implementation of MatrixBlockDataInput - /////////////////////////////////////////////// + /////////////////////////////////////////////// @Override public long readDoubleArray(int len, double[] varr) @@ -244,7 +226,7 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da } } - gnnz += lnnz; + gnnz += lnnz; } } @@ -255,25 +237,15 @@ public class FastBufferedDataInputStream extends FilterInputStream implements Da return nnz; } - private static int baToInt( byte[] ba, final int off ) - { - //shift and add 4 bytes into single int - return ((ba[off+0] & 0xFF) << 24) + - ((ba[off+1] & 0xFF) << 16) + - ((ba[off+2] & 0xFF) << 8) + - ((ba[off+3] & 0xFF) << 0); + private static int baToShort( byte[] ba, final int off ) { + return IOUtilFunctions.baToShort(ba, off); } - private static long baToLong( byte[] ba, final int off ) - { - //shift and add 8 bytes into single long - return ((long)(ba[off+0] & 0xFF) << 56) + - ((long)(ba[off+1] & 0xFF) << 48) + - ((long)(ba[off+2] & 0xFF) << 40) + - ((long)(ba[off+3] & 0xFF) << 32) + - ((long)(ba[off+4] & 0xFF) << 24) + - ((long)(ba[off+5] & 0xFF) << 16) + - ((long)(ba[off+6] & 0xFF) << 8) + - ((long)(ba[off+7] & 0xFF) << 0); + private static int baToInt( byte[] ba, final int off ) { + return IOUtilFunctions.baToInt(ba, off); + } + + private static long baToLong( byte[] ba, final int off ) { + return IOUtilFunctions.baToLong(ba, off); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java index 6173587..dd70d8f 100644 --- a/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java +++ b/src/main/java/org/apache/sysml/runtime/util/FastBufferedDataOutputStream.java @@ -45,34 +45,27 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements protected byte[] _buff; protected int _bufflen; protected int _count; - - public FastBufferedDataOutputStream(OutputStream out) - { + + public FastBufferedDataOutputStream(OutputStream out) { this(out, 8192); } - public FastBufferedDataOutputStream(OutputStream out, int size) - { + public FastBufferedDataOutputStream(OutputStream out, int size) { super(out); - - if(size <= 0) - throw new IllegalArgumentException("Buffer size <= 0."); - if( size%8 != 0 ) - throw new IllegalArgumentException("Buffer size not a multiple of 8."); - + if(size <= 0) + throw new IllegalArgumentException("Buffer size <= 0."); + if( size%8 != 0 ) + throw new IllegalArgumentException("Buffer size not a multiple of 8."); _buff = new byte[size]; _bufflen = size; } @Override - public void write(int b) - throws IOException - { - if (_count >= _bufflen) { - flushBuffer(); - } + public void write(int b) throws IOException { + if (_count >= _bufflen) + flushBuffer(); _buff[_count++] = (byte)b; - } + } @Override public void write(byte[] b, int off, int len) @@ -90,99 +83,74 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements _count += len; } - @Override - public void flush() - throws IOException - { - flushBuffer(); - out.flush(); - } - - private void flushBuffer() - throws IOException - { - if(_count > 0) - { - out.write(_buff, 0, _count); - _count = 0; - } - } - - @Override - public void close() - throws IOException - { - super.close(); - } - - ///////////////////////////// - // DataOutput Implementation - ///////////////////////////// - @Override - public void writeBoolean(boolean v) - throws IOException - { - if (_count >= _bufflen) { - flushBuffer(); + public void flush() throws IOException { + flushBuffer(); + out.flush(); + } + + private void flushBuffer() throws IOException { + if(_count > 0) { + out.write(_buff, 0, _count); + _count = 0; } + } + + @Override + public void close() throws IOException { + super.close(); + } + + ///////////////////////////// + // DataOutput Implementation + ///////////////////////////// + + @Override + public void writeBoolean(boolean v) throws IOException { + if (_count >= _bufflen) + flushBuffer(); _buff[_count++] = (byte)(v ? 1 : 0); } @Override - public void writeInt(int v) - throws IOException - { - if (_count+4 > _bufflen) { - flushBuffer(); - } - + public void writeInt(int v) throws IOException { + if (_count+4 > _bufflen) + flushBuffer(); intToBa(v, _buff, _count); _count += 4; } - @Override - public void writeLong(long v) - throws IOException - { - if (_count+8 > _bufflen) { - flushBuffer(); - } - + public void writeLong(long v) throws IOException { + if (_count+8 > _bufflen) + flushBuffer(); longToBa(v, _buff, _count); _count += 8; } @Override - public void writeDouble(double v) - throws IOException - { - if (_count+8 > _bufflen) { - flushBuffer(); - } - - long tmp = Double.doubleToRawLongBits(v); + public void writeDouble(double v) throws IOException { + if (_count+8 > _bufflen) + flushBuffer(); + long tmp = Double.doubleToRawLongBits(v); longToBa(tmp, _buff, _count); _count += 8; } @Override public void writeByte(int v) throws IOException { - if (_count+1 > _bufflen) { - flushBuffer(); - } - _buff[_count++] = (byte) v; + if (_count+1 > _bufflen) + flushBuffer(); + _buff[_count++] = (byte) v; } @Override public void writeShort(int v) throws IOException { - if (_count+2 > _bufflen) { - flushBuffer(); - } + if (_count+2 > _bufflen) + flushBuffer(); shortToBa(v, _buff, _count); - _count += 2; + _count += 2; } @Override @@ -192,7 +160,7 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements @Override public void writeChar(int v) throws IOException { - throw new IOException("Not supported."); + writeShort(v); } @Override @@ -234,9 +202,9 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements } } - /////////////////////////////////////////////// - // Implementation of MatrixBlockDSMDataOutput - /////////////////////////////////////////////// + /////////////////////////////////////////////// + // Implementation of MatrixBlockDSMDataOutput + /////////////////////////////////////////////// @Override public void writeDoubleArray(int len, double[] varr) @@ -320,32 +288,15 @@ public class FastBufferedDataOutputStream extends FilterOutputStream implements writeInt( 0 ); } - private static void shortToBa( final int val, byte[] ba, final int off ) - { - //shift and mask out 2 bytes - ba[ off+0 ] = (byte)((val >>> 8) & 0xFF); - ba[ off+1 ] = (byte)((val >>> 0) & 0xFF); + private static void shortToBa( final int val, byte[] ba, final int off ) { + IOUtilFunctions.shortToBa(val, ba, off); } - private static void intToBa( final int val, byte[] ba, final int off ) - { - //shift and mask out 4 bytes - ba[ off+0 ] = (byte)((val >>> 24) & 0xFF); - ba[ off+1 ] = (byte)((val >>> 16) & 0xFF); - ba[ off+2 ] = (byte)((val >>> 8) & 0xFF); - ba[ off+3 ] = (byte)((val >>> 0) & 0xFF); + private static void intToBa( final int val, byte[] ba, final int off ) { + IOUtilFunctions.intToBa(val, ba, off); } - private static void longToBa( final long val, byte[] ba, final int off ) - { - //shift and mask out 8 bytes - ba[ off+0 ] = (byte)((val >>> 56) & 0xFF); - ba[ off+1 ] = (byte)((val >>> 48) & 0xFF); - ba[ off+2 ] = (byte)((val >>> 40) & 0xFF); - ba[ off+3 ] = (byte)((val >>> 32) & 0xFF); - ba[ off+4 ] = (byte)((val >>> 24) & 0xFF); - ba[ off+5 ] = (byte)((val >>> 16) & 0xFF); - ba[ off+6 ] = (byte)((val >>> 8) & 0xFF); - ba[ off+7 ] = (byte)((val >>> 0) & 0xFF); + private static void longToBa( final long val, byte[] ba, final int off ) { + IOUtilFunctions.longToBa(val, ba, off); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/68a7b44b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java index a71294a..a40b0b7 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CompressedOuterProductTest.java @@ -35,7 +35,7 @@ import org.apache.sysml.test.integration.TestConfiguration; import org.apache.sysml.test.utils.TestUtils; public class CompressedOuterProductTest extends AutomatedTestBase -{ +{ private static final String TEST_NAME1 = "CompressedOuterProductMain"; private static final String TEST_DIR = "functions/codegen/"; private static final String TEST_CLASS_DIR = TEST_DIR + CompressedOuterProductTest.class.getSimpleName() + "/"; @@ -210,11 +210,11 @@ public class CompressedOuterProductTest extends AutomatedTestBase String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "-stats", + programArgs = new String[]{"-explain", "-stats", "-args", input("X"), output("R") }; fullRScriptName = HOME + testname + ".R"; - rCmd = getRCmd(inputDir(), expectedDir()); + rCmd = getRCmd(inputDir(), expectedDir()); //generate input data double sparsity = -1; @@ -254,7 +254,7 @@ public class CompressedOuterProductTest extends AutomatedTestBase OptimizerUtils.ALLOW_OPERATOR_FUSION = true; CompressedMatrixBlock.ALLOW_DDC_ENCODING = true; } - } + } /** * Override default configuration with custom test configuration to ensure
