Repository: incubator-systemml Updated Branches: refs/heads/master 691cbb801 -> e5aaaf1e8
[SYSTEMML-573] Data-type-specific write of metadata file (frame support) Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/2c19736c Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/2c19736c Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/2c19736c Branch: refs/heads/master Commit: 2c19736c4b35962f5a1c134049df53217cb04590 Parents: 691cbb8 Author: Matthias Boehm <[email protected]> Authored: Fri Apr 22 16:40:37 2016 -0700 Committer: Matthias Boehm <[email protected]> Committed: Fri Apr 22 21:51:02 2016 -0700 ---------------------------------------------------------------------- .../controlprogram/caching/CacheableData.java | 2 +- .../sysml/runtime/util/MapReduceTool.java | 35 +++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2c19736c/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java index d2811a0..7e32cdb 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java @@ -1039,7 +1039,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data { mc = new MatrixCharacteristics(mc.getRows(), mc.getCols(), ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), mc.getNonZeros()); } - MapReduceTool.writeMetaDataFile (filePathAndName + ".mtd", valueType, mc, oinfo, formatProperties); + MapReduceTool.writeMetaDataFile (filePathAndName + ".mtd", valueType, dataType, mc, oinfo, formatProperties); } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/2c19736c/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java index 9b70106..53caea9 100644 --- a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java +++ b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java @@ -40,6 +40,7 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.parser.DataExpression; +import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.io.MatrixReader; @@ -437,12 +438,25 @@ public class MapReduceTool return stats; } - public static void writeMetaDataFile ( String mtdfile, ValueType v, MatrixCharacteristics mc, OutputInfo outinfo) throws IOException { - writeMetaDataFile(mtdfile, v, mc, outinfo, null); + public static void writeMetaDataFile(String mtdfile, ValueType vt, MatrixCharacteristics mc, OutputInfo outinfo) + throws IOException { + writeMetaDataFile(mtdfile, vt, DataType.MATRIX, mc, outinfo); + } + + public static void writeMetaDataFile(String mtdfile, ValueType vt, DataType dt, MatrixCharacteristics mc, OutputInfo outinfo) + throws IOException { + writeMetaDataFile(mtdfile, vt, dt, mc, outinfo, null); } - public static void writeMetaDataFile(String mtdfile, ValueType v, MatrixCharacteristics mc, OutputInfo outinfo, - FileFormatProperties formatProperties) throws IOException { + public static void writeMetaDataFile(String mtdfile, ValueType vt, MatrixCharacteristics mc, OutputInfo outinfo, FileFormatProperties formatProperties) + throws IOException { + writeMetaDataFile(mtdfile, vt, DataType.MATRIX, mc, outinfo, formatProperties); + } + + public static void writeMetaDataFile(String mtdfile, ValueType vt, DataType dt, MatrixCharacteristics mc, + OutputInfo outinfo, FileFormatProperties formatProperties) + throws IOException + { Path pt = new Path(mtdfile); FileSystem fs = FileSystem.get(_rJob); BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(pt,true))); @@ -452,8 +466,8 @@ public class MapReduceTool try { // build JSON metadata object - mtd.put(DataExpression.DATATYPEPARAM, "matrix"); - switch (v) { + mtd.put(DataExpression.DATATYPEPARAM, dt.toString().toLowerCase()); + switch (vt) { case DOUBLE: mtd.put(DataExpression.VALUETYPEPARAM, "double"); break; @@ -475,12 +489,15 @@ public class MapReduceTool } mtd.put(DataExpression.READROWPARAM, mc.getRows()); mtd.put(DataExpression.READCOLPARAM, mc.getCols()); - // only output rows_in_block and cols_in_block for binary format - if (outinfo == OutputInfo.BinaryBlockOutputInfo) { + // only output rows_in_block and cols_in_block for matrix binary format + if (outinfo == OutputInfo.BinaryBlockOutputInfo && dt.isMatrix() ) { mtd.put(DataExpression.ROWBLOCKCOUNTPARAM, mc.getRowsPerBlock()); mtd.put(DataExpression.COLUMNBLOCKCOUNTPARAM, mc.getColsPerBlock()); } - mtd.put(DataExpression.READNUMNONZEROPARAM, mc.getNonZeros()); + // only output nnz for matrix + if( dt.isMatrix() ) { + mtd.put(DataExpression.READNUMNONZEROPARAM, mc.getNonZeros()); + } if (outinfo == OutputInfo.TextCellOutputInfo) { mtd.put(DataExpression.FORMAT_TYPE, "text"); } else if (outinfo == OutputInfo.BinaryBlockOutputInfo || outinfo == OutputInfo.BinaryCellOutputInfo ) {
