Repository: incubator-systemml Updated Branches: refs/heads/master df090f2b1 -> 6758f9034
[SYSTEMML-914] Fix old mlcontext frame output handling (avoid cast2mtx) Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/6758f903 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/6758f903 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/6758f903 Branch: refs/heads/master Commit: 6758f9034f2163ae8eb8d13b700594ee06963481 Parents: df090f2 Author: Matthias Boehm <mbo...@us.ibm.com> Authored: Fri Sep 16 20:02:56 2016 +0200 Committer: Matthias Boehm <mbo...@us.ibm.com> Committed: Fri Sep 16 20:02:56 2016 +0200 ---------------------------------------------------------------------- .../java/org/apache/sysml/api/MLContext.java | 44 ++++++-------------- .../java/org/apache/sysml/api/MLOutput.java | 4 -- 2 files changed, 12 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6758f903/src/main/java/org/apache/sysml/api/MLContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/MLContext.java b/src/main/java/org/apache/sysml/api/MLContext.java index 1e415f1..ced3998 100644 --- a/src/main/java/org/apache/sysml/api/MLContext.java +++ b/src/main/java/org/apache/sysml/api/MLContext.java @@ -1489,53 +1489,34 @@ public class MLContext { // Set active MLContext. _activeMLContext = this; - if(_monitorUtils != null) { _monitorUtils.resetMonitoringData(); } - if(DMLScript.rtplatform == RUNTIME_PLATFORM.SPARK || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK) { - - Map<String, JavaPairRDD<?,?>> retVal = null; - + if( OptimizerUtils.isSparkExecutionMode() ) { // Depending on whether registerInput/registerOutput was called initialize the variables - String[] inputs; String[] outputs; - if(_inVarnames != null) { - inputs = _inVarnames.toArray(new String[0]); - } - else { - inputs = new String[0]; - } - if(_outVarnames != null) { - outputs = _outVarnames.toArray(new String[0]); - } - else { - outputs = new String[0]; - } + String[] inputs = (_inVarnames != null) ? _inVarnames.toArray(new String[0]) : new String[0]; + String[] outputs = (_outVarnames != null) ? _outVarnames.toArray(new String[0]) : new String[0]; + Map<String, JavaPairRDD<?,?>> retVal = (_outVarnames!=null && !_outVarnames.isEmpty()) ? + retVal = new HashMap<String, JavaPairRDD<?,?>>() : null; Map<String, MatrixCharacteristics> outMetadata = new HashMap<String, MatrixCharacteristics>(); - Map<String, String> argVals = DMLScript.createArgumentsMap(isNamedArgument, args); // Run the DML script ExecutionContext ec = executeUsingSimplifiedCompilationChain(dmlScriptFilePath, isFile, argVals, isPyDML, inputs, outputs, _variables, configFilePath); + SparkExecutionContext sec = (SparkExecutionContext) ec; // Now collect the output if(_outVarnames != null) { - if(_variables == null) { - throw new DMLRuntimeException("The symbol table returned after executing the script is empty"); - } + if(_variables == null) + throw new DMLRuntimeException("The symbol table returned after executing the script is empty"); for( String ovar : _outVarnames ) { - if( _variables.keySet().contains(ovar) ) { - if(retVal == null) { - retVal = new HashMap<String, JavaPairRDD<?,?>>(); - } - retVal.put(ovar, ((SparkExecutionContext) ec).getBinaryBlockRDDHandleForVariable(ovar)); - outMetadata.put(ovar, ec.getMatrixCharacteristics(ovar)); // For converting output to dataframe - } - else { + if( !_variables.keySet().contains(ovar) ) throw new DMLException("Error: The variable " + ovar + " is not available as output after the execution of the DMLScript."); - } + + retVal.put(ovar, sec.getRDDHandleForVariable(ovar, InputInfo.BinaryBlockInputInfo)); + outMetadata.put(ovar, ec.getMatrixCharacteristics(ovar)); // For converting output to dataframe } } @@ -1544,7 +1525,6 @@ public class MLContext { else { throw new DMLRuntimeException("Unsupported runtime:" + DMLScript.rtplatform.name()); } - } finally { // Remove global dml config and all thread-local configs http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6758f903/src/main/java/org/apache/sysml/api/MLOutput.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/MLOutput.java b/src/main/java/org/apache/sysml/api/MLOutput.java index d011104..5612392 100644 --- a/src/main/java/org/apache/sysml/api/MLOutput.java +++ b/src/main/java/org/apache/sysml/api/MLOutput.java @@ -159,8 +159,6 @@ public class MLOutput { } public JavaRDD<String> getStringFrameRDD(String varName, String format, CSVFileFormatProperties fprop ) throws DMLRuntimeException { - //TODO MB: note that on construction of MLOutput only matrix binary blocks are passed, and - //hence we will never find a frame binary block in the outputs. JavaPairRDD<Long, FrameBlock> binaryRDD = getFrameBinaryBlockedRDD(varName); MatrixCharacteristics mcIn = getMatrixCharacteristics(varName); if(format.equals("csv")) { @@ -176,8 +174,6 @@ public class MLOutput { } public DataFrame getDataFrameRDD(String varName, JavaSparkContext jsc) throws DMLRuntimeException { - //TODO MB: note that on construction of MLOutput only matrix binary blocks are passed, and - //hence we will never find a frame binary block in the outputs. JavaPairRDD<Long, FrameBlock> binaryRDD = getFrameBinaryBlockedRDD(varName); MatrixCharacteristics mcIn = getMatrixCharacteristics(varName); return FrameRDDConverterUtils.binaryBlockToDataFrame(new SQLContext(jsc), binaryRDD, mcIn, null);
