[SYSTEMML-1851] Fix transformdecode output column names (from meta) This patch fixes the output schema handling of spark and cp transformdecode instructions. For transformdecode the output column names were always set to default names (e.g., C1, C2, etc) because the input is a matrix. We now properly initialize the output column names, based on the column names of the provided meta data frame.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/fc7fcb0a Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/fc7fcb0a Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/fc7fcb0a Branch: refs/heads/master Commit: fc7fcb0aa879feae1c71f7dc98f00b21a4a53f18 Parents: 4c3eab8 Author: Matthias Boehm <[email protected]> Authored: Thu Aug 17 22:06:04 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Fri Aug 18 14:15:43 2017 -0700 ---------------------------------------------------------------------- .../cp/ParameterizedBuiltinCPInstruction.java | 2 ++ .../spark/ParameterizedBuiltinSPInstruction.java | 6 ++++-- .../sysml/runtime/transform/decode/Decoder.java | 17 +++++++++++++---- .../runtime/transform/decode/DecoderFactory.java | 1 + 4 files changed, 20 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/fc7fcb0a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java index 8a3d517..d4df8b0 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ParameterizedBuiltinCPInstruction.java @@ -19,6 +19,7 @@ package org.apache.sysml.runtime.instructions.cp; +import java.util.Arrays; import java.util.HashMap; import org.apache.sysml.lops.Lop; @@ -276,6 +277,7 @@ public class ParameterizedBuiltinCPInstruction extends ComputationCPInstruction //compute transformdecode Decoder decoder = DecoderFactory.createDecoder(getParameterMap().get("spec"), colnames, null, meta); FrameBlock fbout = decoder.decode(data, new FrameBlock(decoder.getSchema())); + fbout.setColumnNames(Arrays.copyOfRange(colnames, 0, fbout.getNumColumns())); //release locks ec.setFrameOutput(output.getName(), fbout); http://git-wip-us.apache.org/repos/asf/systemml/blob/fc7fcb0a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java index 01f4512..1d4b96b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ParameterizedBuiltinSPInstruction.java @@ -20,6 +20,7 @@ package org.apache.sysml.runtime.instructions.spark; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; @@ -844,8 +845,9 @@ public class ParameterizedBuiltinSPInstruction extends ComputationSPInstruction throws Exception { long rix = UtilFunctions.computeCellIndex(in._1().getRowIndex(), _brlen, 0); - return new Tuple2<Long, FrameBlock>(rix, - _decoder.decode(in._2(), new FrameBlock(_decoder.getSchema()))); + FrameBlock fbout = _decoder.decode(in._2(), new FrameBlock(_decoder.getSchema())); + fbout.setColumnNames(Arrays.copyOfRange(_decoder.getColnames(), 0, fbout.getNumColumns())); + return new Tuple2<Long, FrameBlock>(rix, fbout); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/fc7fcb0a/src/main/java/org/apache/sysml/runtime/transform/decode/Decoder.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/transform/decode/Decoder.java b/src/main/java/org/apache/sysml/runtime/transform/decode/Decoder.java index c9d8ab0..2d3f192 100644 --- a/src/main/java/org/apache/sysml/runtime/transform/decode/Decoder.java +++ b/src/main/java/org/apache/sysml/runtime/transform/decode/Decoder.java @@ -34,10 +34,11 @@ public abstract class Decoder implements Serializable { private static final long serialVersionUID = -1732411001366177787L; - protected ValueType[] _schema = null; - protected int[] _colList = null; - - protected Decoder( ValueType[] schema, int[] colList ) { + protected final ValueType[] _schema; + protected final int[] _colList; + protected String[] _colnames = null; + + protected Decoder(ValueType[] schema, int[] colList) { _schema = schema; _colList = colList; } @@ -46,6 +47,14 @@ public abstract class Decoder implements Serializable return _schema; } + public void setColnames(String[] colnames) { + _colnames = colnames; + } + + public String[] getColnames() { + return _colnames; + } + /** * Block decode API converting a matrix block into a frame block. * http://git-wip-us.apache.org/repos/asf/systemml/blob/fc7fcb0a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java index c02609a..2e3dc16 100644 --- a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java +++ b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java @@ -82,6 +82,7 @@ public class DecoderFactory //create composite decoder of all created decoders //and initialize with given meta data (recode, dummy, bin) decoder = new DecoderComposite(schema, ldecoders); + decoder.setColnames(colnames); if( meta != null ) decoder.initMetaData(meta); }
