[SYSTEMML-2157] Fix parfor optimizer side effect on codegen plans The parfor optimizer applies a variety of rewrites some which require the recompilation of the parfor body program or parts of it. Most of these recompilations leverage the recompiler and thus work properly with codegen. However, there are two rewrites (removal of branches and the injection of spark checkpoints) that need to recreate not just instructions but a partial runtime program. This code path was not integrated with codegen yet and hence led to lost fusion plans whenever these rewrites triggered (e.g., for Kmeans over a specific range of data sizes). This patch fixes the underlying code path for partial program recompilations, which ensures robustness independent of concrete instances of rewrites. We also modified the Kmeans tests to check for properly compiled row-wise templates in the inner loop.
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/2c9418c3 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/2c9418c3 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/2c9418c3 Branch: refs/heads/master Commit: 2c9418c3ec6d81dbb0b11c05e56d2672f1992edd Parents: b0fff8c Author: Matthias Boehm <mboe...@gmail.com> Authored: Mon Feb 19 20:06:10 2018 -0800 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Mon Feb 19 20:06:46 2018 -0800 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/parser/DMLTranslator.java | 6 ++++++ .../controlprogram/parfor/opt/ProgramRecompiler.java | 9 +++++++++ .../integration/functions/codegenalg/AlgorithmKMeans.java | 3 ++- 3 files changed, 17 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/2c9418c3/src/main/java/org/apache/sysml/parser/DMLTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java index 8437974..63c896c 100644 --- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java @@ -308,6 +308,12 @@ public class DMLTranslator SpoofCompiler.generateCode(rtprog); } + public void codgenHopsDAG(ProgramBlock pb) + throws HopsException, DMLRuntimeException, LopsException, IOException + { + SpoofCompiler.generateCodeFromProgramBlock(pb); + } + public void constructLops(DMLProgram dmlp) throws ParseException, LanguageException, HopsException, LopsException { // for each namespace, handle function program blocks handle function for( String namespaceKey : dmlp.getNamespaces().keySet() ) http://git-wip-us.apache.org/repos/asf/systemml/blob/2c9418c3/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java index d16d708..3bb0c93 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/ProgramRecompiler.java @@ -28,6 +28,8 @@ import org.apache.sysml.hops.Hop; import org.apache.sysml.hops.HopsException; import org.apache.sysml.hops.IndexingOp; import org.apache.sysml.hops.OptimizerUtils; +import org.apache.sysml.hops.codegen.SpoofCompiler; +import org.apache.sysml.hops.codegen.SpoofCompiler.IntegrationType; import org.apache.sysml.hops.recompile.Recompiler; import org.apache.sysml.lops.Lop; import org.apache.sysml.lops.LopProperties; @@ -74,6 +76,13 @@ public class ProgramRecompiler ret.add(dmlt.createRuntimeProgramBlock(rtprog, sb, config)); } + //enhance runtime program by automatic operator fusion + if( ConfigurationManager.isCodegenEnabled() + && SpoofCompiler.INTEGRATION==IntegrationType.RUNTIME ) { + for( ProgramBlock pb : ret ) + dmlt.codgenHopsDAG(pb); + } + return ret; } http://git-wip-us.apache.org/repos/asf/systemml/blob/2c9418c3/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmKMeans.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmKMeans.java b/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmKMeans.java index c131ea8..f04d684 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmKMeans.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegenalg/AlgorithmKMeans.java @@ -172,7 +172,8 @@ public class AlgorithmKMeans extends AutomatedTestBase runTest(true, false, null, -1); - Assert.assertTrue(heavyHittersContainsSubString("spoof") || heavyHittersContainsSubString("sp_spoof")); + Assert.assertTrue(heavyHittersContainsSubString("spoofCell") || heavyHittersContainsSubString("sp_spoofCell")); + Assert.assertTrue(heavyHittersContainsSubString("spoofRA") || heavyHittersContainsSubString("sp_spoofRA")); } finally { rtplatform = platformOld;