Repository: incubator-systemml Updated Branches: refs/heads/master 0827a4638 -> 7c12992af
[SYSTEMML-864] Fix nrow/ncol operations over csv frame w/ unknown size Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/7c12992a Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/7c12992a Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/7c12992a Branch: refs/heads/master Commit: 7c12992af95b013b9380acd3a6843ee89fae7a3e Parents: 0827a46 Author: Matthias Boehm <[email protected]> Authored: Mon Aug 22 10:51:50 2016 -0700 Committer: Matthias Boehm <[email protected]> Committed: Mon Aug 22 14:38:35 2016 -0700 ---------------------------------------------------------------------- .../org/apache/sysml/hops/OptimizerUtils.java | 9 +++++++++ .../cp/AggregateUnaryCPInstruction.java | 20 +++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7c12992a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java index 8e53e96..52ee378 100644 --- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java +++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java @@ -594,6 +594,15 @@ public class OptimizerUtils * * @return */ + public static boolean isHadoopExecutionMode() { + return ( DMLScript.rtplatform == RUNTIME_PLATFORM.HADOOP + || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID); + } + + /** + * + * @return + */ public static boolean isHybridExecutionMode() { return ( DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID || DMLScript.rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK ); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/7c12992a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java index 24b2438..bafaeb2 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java @@ -21,9 +21,10 @@ package org.apache.sysml.runtime.instructions.cp; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysml.runtime.controlprogram.caching.CacheableData; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.functionobjects.Builtin; import org.apache.sysml.runtime.instructions.InstructionUtils; @@ -101,13 +102,18 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction //Note: check on matrix characteristics to cover incorrect length (-1*-1 -> 1) if( !mc.dimsKnown() ) //invalid nrow/ncol/length { - if( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE ) + if( DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE + || (input1.getDataType() == DataType.FRAME && OptimizerUtils.isHadoopExecutionMode()) ) { - //read the input data and explicitly refresh input data - MatrixObject mo = (MatrixObject)ec.getVariable(input1.getName()); - mo.acquireRead(); - mo.refreshMetaData(); - mo.release(); + if( OptimizerUtils.isHadoopExecutionMode() ) { + LOG.warn("Reading csv input frame of unkown size into memory for '"+opcode+"'."); + } + + //read the input matrix/frame and explicitly refresh meta data + CacheableData<?> obj = ec.getCacheableData(input1.getName()); + obj.acquireRead(); + obj.refreshMetaData(); + obj.release(); //update meta data information mc = ec.getMatrixCharacteristics(input1.getName());
