Repository: systemml Updated Branches: refs/heads/master 5ed2c30e6 -> ca615ca10
[SYSTEMML-2222] Improved empty block filtering for spark operations This patch improves the analysis of safe empty block filtering for spark operations during hop compilation. We now handle mixed chains of matrix multiply and reorg operations. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/80023787 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/80023787 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/80023787 Branch: refs/heads/master Commit: 80023787e9a4299468759c1a2a2e37fc6dd58054 Parents: 5ed2c30 Author: Matthias Boehm <mboe...@gmail.com> Authored: Fri Apr 13 14:10:23 2018 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Fri Apr 13 14:10:23 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/sysml/hops/OptimizerUtils.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/80023787/src/main/java/org/apache/sysml/hops/OptimizerUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java index 3156afd..5ae20eb 100644 --- a/src/main/java/org/apache/sysml/hops/OptimizerUtils.java +++ b/src/main/java/org/apache/sysml/hops/OptimizerUtils.java @@ -33,6 +33,7 @@ import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.hops.Hop.DataOpTypes; import org.apache.sysml.hops.Hop.FileFormatTypes; import org.apache.sysml.hops.Hop.OpOp2; +import org.apache.sysml.hops.Hop.ReOrgOp; import org.apache.sysml.hops.rewrite.HopRewriteUtils; import org.apache.sysml.lops.Checkpoint; import org.apache.sysml.lops.Lop; @@ -922,13 +923,13 @@ public class OptimizerUtils boolean ret = true; for( Hop p : hop.getParent() ) { p.optFindExecType(); //ensure exec type evaluated - ret &= ( p.getExecType()==ExecType.CP - ||(p instanceof AggBinaryOp && allowsToFilterEmptyBlockOutputs(p) ) - ||(p instanceof DataOp && ((DataOp)p).getDataOpType()==DataOpTypes.PERSISTENTWRITE && ((DataOp)p).getInputFormatType()==FileFormatTypes.TEXT)) - && !(p instanceof FunctionOp || (p instanceof DataOp && ((DataOp)p).getInputFormatType()!=FileFormatTypes.TEXT) ); //no function call or transient write + ret &= ( p.getExecType()==ExecType.CP + ||(p instanceof AggBinaryOp && allowsToFilterEmptyBlockOutputs(p) ) + ||(HopRewriteUtils.isReorg(p, ReOrgOp.RESHAPE, ReOrgOp.TRANS) && allowsToFilterEmptyBlockOutputs(p) ) + ||(HopRewriteUtils.isData(p, DataOpTypes.PERSISTENTWRITE) && ((DataOp)p).getInputFormatType()==FileFormatTypes.TEXT)) + && !(p instanceof FunctionOp || (p instanceof DataOp && ((DataOp)p).getInputFormatType()!=FileFormatTypes.TEXT) ); //no function call or transient write } - - return ret; + return ret; } public static int getConstrainedNumThreads(int maxNumThreads)
