[SYSTEMML-1126] Fix parfor spark/mr handling of custom compiler configs This patch fixes an issue that has been introduced with SYSTEMML-584 and affects, in the context of parfor spark/mr execute jobs, all configurations that are taken over from the dml config into our compiler config (i.e., optimization level and blocksize).
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/e82de90b Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/e82de90b Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/e82de90b Branch: refs/heads/master Commit: e82de90b21df44a3194a86ce0e087dc710cd34af Parents: baa70a1 Author: Matthias Boehm <[email protected]> Authored: Fri Feb 24 17:50:16 2017 -0800 Committer: Matthias Boehm <[email protected]> Committed: Sat Feb 25 11:51:03 2017 -0800 ---------------------------------------------------------------------- .../runtime/controlprogram/parfor/ProgramConverter.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/e82de90b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java index f94be23..5cc9ca1 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java @@ -31,9 +31,11 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.mapred.JobConf; import org.apache.sysml.api.DMLScript; import org.apache.sysml.conf.CompilerConfig.ConfigType; +import org.apache.sysml.conf.CompilerConfig; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.hops.Hop; +import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.hops.recompile.Recompiler; import org.apache.sysml.parser.DMLProgram; import org.apache.sysml.parser.DataIdentifier; @@ -1345,8 +1347,10 @@ public class ProgramConverter JobConf job = ConfigurationManager.getCachedJobConf(); if( !InfrastructureAnalyzer.isLocalMode(job) ) { if( confStr != null && !confStr.trim().isEmpty() ) { - DMLConfig config = DMLConfig.parseDMLConfig(confStr); - ConfigurationManager.setLocalConfig(config); + DMLConfig dmlconf = DMLConfig.parseDMLConfig(confStr); + CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf); + ConfigurationManager.setLocalConfig(dmlconf); + ConfigurationManager.setLocalConfig(cconf); } //init internal configuration w/ parsed or default config ParForProgramBlock.initInternalConfigurations(
