Repository: systemml Updated Branches: refs/heads/master 687e19c55 -> 754548190
[SYSTEMML-1900] Allow external codegen java compiler configuration This patch exposes the codegen compiler configuration via a SystemML configuration property. The default is 'auto', which is equivalent to the previous behavior. However, now users can explicitly set the codegen compiler, which is useful for programmatic APIs such as JMLC. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/f45493e8 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/f45493e8 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/f45493e8 Branch: refs/heads/master Commit: f45493e8aabb33ec1f628c016b3e68508dcf4ac8 Parents: 687e19c Author: Matthias Boehm <[email protected]> Authored: Mon Sep 11 14:00:36 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Mon Sep 11 15:20:05 2017 -0700 ---------------------------------------------------------------------- conf/SystemML-config.xml.template | 3 +++ src/main/java/org/apache/sysml/conf/DMLConfig.java | 11 +++++++---- .../org/apache/sysml/hops/codegen/SpoofCompiler.java | 9 ++++++++- 3 files changed, 18 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/f45493e8/conf/SystemML-config.xml.template ---------------------------------------------------------------------- diff --git a/conf/SystemML-config.xml.template b/conf/SystemML-config.xml.template index 86bacda..454d0cc 100644 --- a/conf/SystemML-config.xml.template +++ b/conf/SystemML-config.xml.template @@ -60,6 +60,9 @@ <!-- enables operator fusion via code generation, experimental feature --> <codegen.enabled>false</codegen.enabled> + <!-- set the codegen java compiler (auto, janino, javac) --> + <codegen.compiler>auto</codegen.compiler> + <!-- if codegen.enabled, enables source code caching of fused operators --> <codegen.plancache>false</codegen.plancache> http://git-wip-us.apache.org/repos/asf/systemml/blob/f45493e8/src/main/java/org/apache/sysml/conf/DMLConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java index 84beea8..bfb3850 100644 --- a/src/main/java/org/apache/sysml/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java @@ -40,6 +40,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.sysml.hops.OptimizerUtils; +import org.apache.sysml.hops.codegen.SpoofCompiler.CompilerType; import org.apache.sysml.parser.ParseException; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.io.IOUtilFunctions; @@ -74,6 +75,7 @@ public class DMLConfig public static final String COMPRESSED_LINALG = "compressed.linalg"; public static final String NATIVE_BLAS = "native.blas"; public static final String CODEGEN = "codegen.enabled"; //boolean + public static final String CODEGEN_COMPILER = "codegen.compiler"; //see SpoofCompiler.CompilerType public static final String CODEGEN_PLANCACHE = "codegen.plancache"; //boolean public static final String CODEGEN_LITERALS = "codegen.literals"; //1..heuristic, 2..always public static final String EXTRA_FINEGRAINED_STATS = "systemml.stats.finegrained"; //boolean @@ -119,16 +121,16 @@ public class DMLConfig _defaultVals.put(CP_PARALLEL_IO, "true" ); _defaultVals.put(COMPRESSED_LINALG, "false" ); _defaultVals.put(CODEGEN, "false" ); + _defaultVals.put(CODEGEN_COMPILER, CompilerType.AUTO.name() ); _defaultVals.put(CODEGEN_PLANCACHE, "true" ); _defaultVals.put(CODEGEN_LITERALS, "1" ); _defaultVals.put(NATIVE_BLAS, "none" ); _defaultVals.put(EXTRA_FINEGRAINED_STATS,"false" ); - _defaultVals.put(STATS_MAX_WRAP_LEN,"30" ); + _defaultVals.put(STATS_MAX_WRAP_LEN, "30" ); _defaultVals.put(EXTRA_GPU_STATS, "false" ); _defaultVals.put(EXTRA_DNN_STATS, "false" ); - _defaultVals.put(GPU_MEMORY_UTILIZATION_FACTOR, "0.9" ); - _defaultVals.put(AVAILABLE_GPUS, "-1"); + _defaultVals.put(AVAILABLE_GPUS, "-1"); } public DMLConfig() @@ -408,7 +410,8 @@ public class DMLConfig NUM_REDUCERS, DEFAULT_BLOCK_SIZE, YARN_APPMASTER, YARN_APPMASTERMEM, YARN_MAPREDUCEMEM, CP_PARALLEL_OPS, CP_PARALLEL_IO, NATIVE_BLAS, - COMPRESSED_LINALG, CODEGEN, CODEGEN_LITERALS, CODEGEN_PLANCACHE, + COMPRESSED_LINALG, + CODEGEN, CODEGEN_COMPILER, CODEGEN_PLANCACHE, CODEGEN_LITERALS, EXTRA_GPU_STATS, EXTRA_DNN_STATS, EXTRA_FINEGRAINED_STATS, STATS_MAX_WRAP_LEN, AVAILABLE_GPUS }; http://git-wip-us.apache.org/repos/asf/systemml/blob/f45493e8/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java b/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java index ca3c233..b98342c 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java +++ b/src/main/java/org/apache/sysml/hops/codegen/SpoofCompiler.java @@ -36,6 +36,8 @@ import org.apache.log4j.Logger; import org.apache.sysml.api.DMLException; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.conf.ConfigurationManager; +import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.hops.codegen.cplan.CNode; import org.apache.sysml.hops.codegen.cplan.CNodeCell; import org.apache.sysml.hops.codegen.cplan.CNodeData; @@ -114,6 +116,7 @@ public class SpoofCompiler public static final PlanSelector PLAN_SEL_POLICY = PlanSelector.FUSE_COST_BASED_V2; public enum CompilerType { + AUTO, JAVAC, JANINO, } @@ -484,7 +487,11 @@ public class SpoofCompiler } public static void setExecTypeSpecificJavaCompiler() { - JAVA_COMPILER = OptimizerUtils.isSparkExecutionMode() ? + DMLConfig conf = ConfigurationManager.getDMLConfig(); + String compiler = conf.getTextValue(DMLConfig.CODEGEN_COMPILER); + CompilerType type = CompilerType.valueOf(compiler.toUpperCase()); + JAVA_COMPILER = (type != CompilerType.AUTO) ? type : + OptimizerUtils.isSparkExecutionMode() ? CompilerType.JANINO : CompilerType.JAVAC; }
