[SYSTEMML-556] JMLC api extension for input and output frames This patch adds input and output frames to the JMLC API. Frames are specified by a mixed schema or implicit string schema and data is passed as a two dimensional string array or directly as a FrameBlock. Furthermore, this change also includes extended casting functionalities between frames and string arrays.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/5a2d888a Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/5a2d888a Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/5a2d888a Branch: refs/heads/master Commit: 5a2d888a77d9b5acb082c4a63149d96b5b5f5894 Parents: ccfe921 Author: Matthias Boehm <[email protected]> Authored: Thu Mar 10 21:25:31 2016 -0800 Committer: Matthias Boehm <[email protected]> Committed: Thu Mar 10 21:25:31 2016 -0800 ---------------------------------------------------------------------- .../apache/sysml/api/jmlc/PreparedScript.java | 77 +++++++++++++++----- .../apache/sysml/api/jmlc/ResultVariables.java | 31 +++++++- .../controlprogram/caching/FrameObject.java | 4 +- .../sysml/runtime/util/DataConverter.java | 58 +++++++++++++++ 4 files changed, 148 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5a2d888a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java index 381f83c..3bc4fc3 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java +++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map.Entry; import org.apache.sysml.api.DMLException; @@ -31,6 +32,7 @@ import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.controlprogram.LocalVariableMap; import org.apache.sysml.runtime.controlprogram.Program; +import org.apache.sysml.runtime.controlprogram.caching.FrameObject; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory; @@ -42,6 +44,7 @@ import org.apache.sysml.runtime.instructions.cp.ScalarObject; import org.apache.sysml.runtime.instructions.cp.StringObject; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.MatrixFormatMetaData; +import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.OutputInfo; @@ -140,27 +143,13 @@ public class PreparedScript _vars.put(varname, scalar); } - /** - * - * @param varname - * @param matrix - * @throws DMLException - */ - public void setMatrix(String varname, double[][] matrix) - throws DMLException - { + /** Binds a matrix object to a registered input variable. */ + public void setMatrix(String varname, double[][] matrix) throws DMLException { setMatrix(varname, matrix, false); } - /** - * - * @param varname - * @param matrix - * @throws DMLException - */ - public void setMatrix(String varname, double[][] matrix, boolean reuse) - throws DMLException - { + /** Binds a matrix object to a registered input variable. */ + public void setMatrix(String varname, double[][] matrix, boolean reuse) throws DMLException { setMatrix(varname, DataConverter.convertToMatrixBlock(matrix), reuse); } @@ -197,6 +186,58 @@ public class PreparedScript _inVarReuse.put(varname, mo); } } + + /** Binds a frame object to a registered input variable. */ + public void setFrame(String varname, String[][] frame) throws DMLException { + setFrame(varname, frame, false); + } + + /** Binds a frame object to a registered input variable. */ + public void setFrame(String varname, String[][] frame, List<ValueType> schema) throws DMLException { + setFrame(varname, frame, schema, false); + } + + /** Binds a frame object to a registered input variable. */ + public void setFrame(String varname, String[][] frame, boolean reuse) throws DMLException { + setFrame(varname, DataConverter.convertToFrameBlock(frame), reuse); + } + + /** Binds a frame object to a registered input variable. */ + public void setFrame(String varname, String[][] frame, List<ValueType> schema, boolean reuse) throws DMLException { + setFrame(varname, DataConverter.convertToFrameBlock(frame, schema), reuse); + } + + /** + * Binds a frame object to a registered input variable. + * If reuse requested, then the input is guaranteed to be + * preserved over multiple <code>executeScript</code> calls. + * + * @param varname + * @param frame + * @param reuse + * @throws DMLException + */ + public void setFrame(String varname, FrameBlock frame, boolean reuse) + throws DMLException + { + if( !_inVarnames.contains(varname) ) + throw new DMLException("Unspecified input variable: "+varname); + + DMLConfig conf = ConfigurationManager.getConfig(); + String scratch_space = conf.getTextValue(DMLConfig.SCRATCH_SPACE); + + //create new frame object + String fname = scratch_space+"/"+varname; + FrameObject fo = new FrameObject(fname, frame); + + //put create matrix wrapper into symbol table + _vars.put(varname, fo); + if( reuse ) { + //TODO buffer pool integration + //mo.enableCleanup(false); //prevent cleanup + _inVarReuse.put(varname, fo); + } + } /** * Remove all current values bound to input or output variables. http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5a2d888a/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java b/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java index 87910df..a5bf74a 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java +++ b/src/main/java/org/apache/sysml/api/jmlc/ResultVariables.java @@ -23,8 +23,10 @@ import java.util.HashMap; import java.util.Set; import org.apache.sysml.api.DMLException; +import org.apache.sysml.runtime.controlprogram.caching.FrameObject; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; import org.apache.sysml.runtime.instructions.cp.Data; +import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.util.DataConverter; @@ -55,7 +57,7 @@ public class ResultVariables /** * - * @param var + * @param varname * @return * @throws DMLException */ @@ -83,6 +85,33 @@ public class ResultVariables /** * + * @param varname + * @return + * @throws DMLException + */ + public String[][] getFrame(String varname) + throws DMLException + { + if( !_out.containsKey(varname) ) + throw new DMLException("Non-existing output variable: "+varname); + + String[][] ret = null; + Data dat = _out.get(varname); + + //basic checks for data type + if( !(dat instanceof FrameObject) ) + throw new DMLException("Expected frame result '"+varname+"' not a frame."); + + //convert output matrix to double array + FrameObject fo = (FrameObject)dat; + FrameBlock frame = fo.getData(); + ret = DataConverter.convertToStringFrame(frame); + + return ret; + } + + /** + * * * @param ovar * @param data http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5a2d888a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java index 40c62c7..701b8c9 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/FrameObject.java @@ -19,8 +19,6 @@ package org.apache.sysml.runtime.controlprogram.caching; -import java.util.List; - import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.matrix.data.FrameBlock; @@ -40,7 +38,7 @@ public class FrameObject extends CacheableData super(DataType.FRAME, ValueType.UNKNOWN); } - public FrameObject(String fname, List<ValueType> schema, FrameBlock data) { + public FrameObject(String fname, FrameBlock data) { this(); setFileName(fname); setData(data); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/5a2d888a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java index c75aabf..131bee6 100644 --- a/src/main/java/org/apache/sysml/runtime/util/DataConverter.java +++ b/src/main/java/org/apache/sysml/runtime/util/DataConverter.java @@ -642,6 +642,64 @@ public class DataConverter } /** + * Converts a frame block with arbitrary schema into a two dimensional + * string array. + * + * @param frame + * @return + * @throws DMLRuntimeException + */ + public static String[][] convertToStringFrame(FrameBlock frame) + throws DMLRuntimeException + { + String[][] ret = new String[frame.getNumRows()][]; + Iterator<String[]> iter = frame.getStringRowIterator(); + for( int i=0; iter.hasNext(); i++ ) { + //deep copy output rows due to internal reuse + ret[i] = iter.next().clone(); + } + + return ret; + } + + /** + * Converts a two dimensions string array into a frame block of + * value type string. If the given array is null or of length 0, + * we return an empty frame block. + * + * @param data + * @return + */ + public static FrameBlock convertToFrameBlock(String[][] data) { + //check for empty frame block + if( data == null || data.length==0 ) + return new FrameBlock(); + + //construct temporary schema + List<ValueType> schema = new ArrayList<ValueType>(); + for( int j=0; j<data[0].length; j++ ) + schema.add(ValueType.STRING); + + //create frame block + return convertToFrameBlock(data, schema); + } + + /** + * + * @param data + * @param schema + * @return + */ + public static FrameBlock convertToFrameBlock(String[][] data, List<ValueType> schema) { + //check for empty frame block + if( data == null || data.length==0 ) + return new FrameBlock(); + + //create frame block + return new FrameBlock(schema, data); + } + + /** * Converts a matrix block into a frame block of value type double. * * @param mb
