Repository: incubator-systemml Updated Branches: refs/heads/master 4481346f9 -> 8d5f3ceab
[SYSTEMML-925] Performance frame block serialization w/ default metadata Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/0e8f1e18 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/0e8f1e18 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/0e8f1e18 Branch: refs/heads/master Commit: 0e8f1e1863aa53f77220254051b75cc8c3ce670c Parents: 4481346 Author: Matthias Boehm <mbo...@us.ibm.com> Authored: Fri Sep 16 23:17:47 2016 +0200 Committer: Matthias Boehm <mbo...@us.ibm.com> Committed: Sat Sep 17 00:25:16 2016 -0700 ---------------------------------------------------------------------- .../sysml/runtime/matrix/data/FrameBlock.java | 44 ++++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/0e8f1e18/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java index 1c8ffd0..8e68014 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java @@ -305,6 +305,26 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable /** * + * @return + */ + public boolean isColNamesDefault() { + boolean ret = true; + for( int j=0; j<getNumColumns() && ret; j++ ) + ret &= isColNameDefault(j); + return ret; + } + + /** + * + * @param i + * @return + */ + public boolean isColNameDefault(int i) { + return _colnames.get(i).equals("C"+i); + } + + /** + * */ public void recomputeColumnCardinality() { for( int j=0; j<getNumColumns(); j++ ) { @@ -510,16 +530,21 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable @Override public void write(DataOutput out) throws IOException { - //write header (rows, cols) + boolean isDefaultMeta = isColNamesDefault() + && isColumnMetadataDefault(); + //write header (rows, cols, default) out.writeInt(getNumRows()); out.writeInt(getNumColumns()); + out.writeBoolean(isDefaultMeta); //write columns (value type, data) for( int j=0; j<getNumColumns(); j++ ) { out.writeByte(_schema.get(j).ordinal()); - out.writeUTF(_colnames.get(j)); - out.writeLong(_colmeta.get(j).getNumDistinct()); - out.writeUTF( (_colmeta.get(j).getMvValue()!=null) ? - _colmeta.get(j).getMvValue() : "" ); + if( !isDefaultMeta ) { + out.writeUTF(_colnames.get(j)); + out.writeLong(_colmeta.get(j).getNumDistinct()); + out.writeUTF( (_colmeta.get(j).getMvValue()!=null) ? + _colmeta.get(j).getMvValue() : "" ); + } _coldata.get(j).write(out); } } @@ -529,15 +554,16 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable //read head (rows, cols) _numRows = in.readInt(); int numCols = in.readInt(); + boolean isDefaultMeta = in.readBoolean(); //read columns (value type, meta, data) _schema.clear(); _colmeta.clear(); _coldata.clear(); for( int j=0; j<numCols; j++ ) { ValueType vt = ValueType.values()[in.readByte()]; - String name = in.readUTF(); - long ndistinct = in.readLong(); - String mvvalue = in.readUTF(); + String name = isDefaultMeta ? createColName(j) : in.readUTF(); + long ndistinct = isDefaultMeta ? 0 : in.readLong(); + String mvvalue = isDefaultMeta ? null : in.readUTF(); Array arr = null; switch( vt ) { case STRING: arr = new StringArray(new String[_numRows]); break; @@ -550,7 +576,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable _schema.add(vt); _colnames.add(name); _colmeta.add(new ColumnMetadata(ndistinct, - mvvalue.isEmpty() ? null : mvvalue)); + (mvvalue==null || mvvalue.isEmpty()) ? null : mvvalue)); _coldata.add(arr); } }
