Repository: systemml Updated Branches: refs/heads/master 72454cdaa -> a13632d45
[SYSTEMML-2341] Improved recompilation of right indexing and reshape This patch improves special cases of size updates during dynamic recompilation and thus, also during intra- and inter-procedural analysis (IPA). Specifically, we now handle the size reset, recursive DAG propagation, and update with symbol table statistics in a consistent manner. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/fc5039ed Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/fc5039ed Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/fc5039ed Branch: refs/heads/master Commit: fc5039ed7b85949285c9c450dc2635e09214b42d Parents: 72454cd Author: Matthias Boehm <[email protected]> Authored: Wed May 23 17:10:04 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Wed May 23 17:10:04 2018 -0700 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/Hop.java | 4 +- .../apache/sysml/hops/recompile/Recompiler.java | 74 ++++++++------------ .../recompile/FunctionRecompileTest.java | 6 +- 3 files changed, 36 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/fc5039ed/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index 7b0ac5b..de84088 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -1720,11 +1720,11 @@ public abstract class Hop implements ParseInfo return ret; } - public double computeBoundsInformation( Hop input, LocalVariableMap vars ) { + public final double computeBoundsInformation( Hop input, LocalVariableMap vars ) { return computeBoundsInformation(input, vars, new HashMap<Long, Double>()); } - public double computeBoundsInformation( Hop input, LocalVariableMap vars, HashMap<Long, Double> memo ) { + public final double computeBoundsInformation( Hop input, LocalVariableMap vars, HashMap<Long, Double> memo ) { double ret = Double.MAX_VALUE; try { ret = OptimizerUtils.rEvalSimpleDoubleExpression(input, memo, vars); http://git-wip-us.apache.org/repos/asf/systemml/blob/fc5039ed/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java index fe69b49..0364724 100644 --- a/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java +++ b/src/main/java/org/apache/sysml/hops/recompile/Recompiler.java @@ -46,12 +46,12 @@ import org.apache.sysml.hops.Hop.DataGenMethod; import org.apache.sysml.hops.Hop.DataOpTypes; import org.apache.sysml.hops.Hop.FileFormatTypes; import org.apache.sysml.hops.Hop.OpOp1; +import org.apache.sysml.hops.Hop.ReOrgOp; import org.apache.sysml.hops.HopsException; import org.apache.sysml.hops.IndexingOp; import org.apache.sysml.hops.LiteralOp; import org.apache.sysml.hops.MemoTable; import org.apache.sysml.hops.OptimizerUtils; -import org.apache.sysml.hops.ReorgOp; import org.apache.sysml.hops.UnaryOp; import org.apache.sysml.hops.codegen.SpoofCompiler; import org.apache.sysml.hops.rewrite.HopRewriteUtils; @@ -1282,17 +1282,12 @@ public class Recompiler for( Hop c : hop.getInput() ) rUpdateStatistics(c, vars); - boolean updatedSizeExpr = false; - //update statistics for transient reads according to current statistics //(with awareness not to override persistent reads to an existing name) - if( hop instanceof DataOp - && ((DataOp)hop).getDataOpType() != DataOpTypes.PERSISTENTREAD ) - { + if( HopRewriteUtils.isData(hop, DataOpTypes.TRANSIENTREAD) ) { DataOp d = (DataOp) hop; String varName = d.getName(); - if( vars.keySet().contains( varName ) ) - { + if( vars.keySet().contains( varName ) ) { Data dat = vars.get(varName); if( dat instanceof MatrixObject ) { MatrixObject mo = (MatrixObject) dat; @@ -1308,10 +1303,9 @@ public class Recompiler } } //special case for persistent reads with unknown size (read-after-write) - else if( hop instanceof DataOp - && ((DataOp)hop).getDataOpType() == DataOpTypes.PERSISTENTREAD - && !hop.dimsKnown() && ((DataOp)hop).getInputFormatType()!=FileFormatTypes.CSV - && !ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_READ_WRITE_METADATA) ) + else if( HopRewriteUtils.isData(hop, DataOpTypes.PERSISTENTREAD) + && !hop.dimsKnown() && ((DataOp)hop).getInputFormatType()!=FileFormatTypes.CSV + && !ConfigurationManager.getCompilerConfigFlag(ConfigType.IGNORE_READ_WRITE_METADATA) ) { //update hop with read meta data DataOp dop = (DataOp) hop; @@ -1332,7 +1326,8 @@ public class Recompiler HashMap<Long, Long> memo = new HashMap<>(); d.refreshRowsParameterInformation(d.getInput().get(ix1), vars, memo); d.refreshColsParameterInformation(d.getInput().get(ix2), vars, memo); - updatedSizeExpr = initUnknown & d.dimsKnown(); + if( !(initUnknown & d.dimsKnown()) ) + d.refreshSizeInformation(); } else if ( d.getOp() == DataGenMethod.SEQ ) { @@ -1355,47 +1350,40 @@ public class Recompiler d.setDim2( 1 ); d.setIncrementValue( incr ); } - updatedSizeExpr = initUnknown & d.dimsKnown(); + if( !(initUnknown & d.dimsKnown()) ) + d.refreshSizeInformation(); } else { throw new DMLRuntimeException("Unexpected data generation method: " + d.getOp()); } } //update size expression for reshape according to symbol table entries - else if ( hop instanceof ReorgOp - && ((ReorgOp)(hop)).getOp()==Hop.ReOrgOp.RESHAPE ) - { - ReorgOp d = (ReorgOp) hop; - boolean initUnknown = !d.dimsKnown(); - HashMap<Long, Long> memo = new HashMap<>(); - d.refreshRowsParameterInformation(d.getInput().get(1), vars, memo); - d.refreshColsParameterInformation(d.getInput().get(2), vars, memo); - updatedSizeExpr = initUnknown & d.dimsKnown(); + else if( HopRewriteUtils.isReorg(hop, ReOrgOp.RESHAPE) ) { + hop.refreshSizeInformation(); //update incl reset + if( !hop.dimsKnown() ) { + HashMap<Long, Long> memo = new HashMap<>(); + hop.refreshRowsParameterInformation(hop.getInput().get(1), vars, memo); + hop.refreshColsParameterInformation(hop.getInput().get(2), vars, memo); + } } //update size expression for indexing according to symbol table entries - else if( hop instanceof IndexingOp && hop.getDataType()!=DataType.LIST ) - { - IndexingOp iop = (IndexingOp)hop; - Hop input2 = iop.getInput().get(1); //inpRowL - Hop input3 = iop.getInput().get(2); //inpRowU - Hop input4 = iop.getInput().get(3); //inpColL - Hop input5 = iop.getInput().get(4); //inpColU - boolean initUnknown = !iop.dimsKnown(); - HashMap<Long, Double> memo = new HashMap<>(); - double rl = iop.computeBoundsInformation(input2, vars, memo); - double ru = iop.computeBoundsInformation(input3, vars, memo); - double cl = iop.computeBoundsInformation(input4, vars, memo); - double cu = iop.computeBoundsInformation(input5, vars, memo); - if( rl!=Double.MAX_VALUE && ru!=Double.MAX_VALUE ) - iop.setDim1( (long)(ru-rl+1) ); - if( cl!=Double.MAX_VALUE && cu!=Double.MAX_VALUE ) - iop.setDim2( (long)(cu-cl+1) ); - updatedSizeExpr = initUnknown & iop.dimsKnown(); + else if( hop instanceof IndexingOp && hop.getDataType()!=DataType.LIST ) { + hop.refreshSizeInformation(); //update, incl reset + if( !hop.dimsKnown() ) { + HashMap<Long, Double> memo = new HashMap<>(); + double rl = hop.computeBoundsInformation(hop.getInput().get(1), vars, memo); + double ru = hop.computeBoundsInformation(hop.getInput().get(2), vars, memo); + double cl = hop.computeBoundsInformation(hop.getInput().get(3), vars, memo); + double cu = hop.computeBoundsInformation(hop.getInput().get(4), vars, memo); + if( rl!=Double.MAX_VALUE && ru!=Double.MAX_VALUE ) + hop.setDim1( (long)(ru-rl+1) ); + if( cl!=Double.MAX_VALUE && cu!=Double.MAX_VALUE ) + hop.setDim2( (long)(cu-cl+1) ); + } } - + else { //propagate statistics along inner nodes of DAG, //without overwriting inferred size expressions - if( !updatedSizeExpr ) { hop.refreshSizeInformation(); } http://git-wip-us.apache.org/repos/asf/systemml/blob/fc5039ed/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java index 55a2d5b..60f263f 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/recompile/FunctionRecompileTest.java @@ -103,15 +103,15 @@ public class FunctionRecompileTest extends AutomatedTestBase //note: change from previous version due to fix in op selection (unknown size XtX and mapmult) //CHECK compiled MR jobs int expectNumCompiled = -1; - if( IPA ) expectNumCompiled = 4; //reblock TODO investigate 1-4 recompile side effect - else expectNumCompiled = 5;//reblock, GMR,GMR,GMR,GMR (last two should piggybacked) + if( IPA ) expectNumCompiled = 1; //reblock + else expectNumCompiled = 5; //reblock, GMR,GMR,GMR,GMR (last two should piggybacked) Assert.assertEquals("Unexpected number of compiled MR jobs.", expectNumCompiled, Statistics.getNoOfCompiledMRJobs()); //CHECK executed MR jobs int expectNumExecuted = -1; if( recompile ) expectNumExecuted = 0; - else if( IPA ) expectNumExecuted = 31; //reblock TODO investigate 1-31 recompile side effect + else if( IPA ) expectNumExecuted = 1; //reblock else expectNumExecuted = 41; //reblock, 10*(GMR,GMR,GMR, GMR) (last two should piggybacked) Assert.assertEquals("Unexpected number of executed MR jobs.", expectNumExecuted, Statistics.getNoOfExecutedMRJobs());
