Repository: systemml Updated Branches: refs/heads/master cc7ec7625 -> bf4ba16b9
[SYSTEMML-1325] Fixes formatting issues and warnings. Fixes bug causing explain to sometimes not be printed. Closes #838. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/bf4ba16b Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/bf4ba16b Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/bf4ba16b Branch: refs/heads/master Commit: bf4ba16b9aaa9afee20a3f1c03b0ff49c5346a9d Parents: cc7ec76 Author: Anthony Thomas <ahtho...@eng.ucsd.edu> Authored: Sat Nov 3 05:44:54 2018 +0530 Committer: Niketan Pansare <npan...@us.ibm.com> Committed: Sat Nov 3 05:44:54 2018 +0530 ---------------------------------------------------------------------- .../java/org/apache/sysml/api/DMLScript.java | 104 ++++----- .../apache/sysml/api/ScriptExecutorUtils.java | 20 +- .../apache/sysml/api/jmlc/PreparedScript.java | 8 +- .../sysml/api/mlcontext/ScriptExecutor.java | 232 +------------------ .../apache/sysml/conf/ConfigurationManager.java | 8 +- .../controlprogram/LocalVariableMap.java | 1 - .../org/apache/sysml/test/gpu/JMLCTests.java | 186 ++++++++------- .../integration/mlcontext/MLContextTest.java | 10 +- 8 files changed, 170 insertions(+), 399 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/bf4ba16b/src/main/java/org/apache/sysml/api/DMLScript.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java index 16d8986..293a810 100644 --- a/src/main/java/org/apache/sysml/api/DMLScript.java +++ b/src/main/java/org/apache/sysml/api/DMLScript.java @@ -376,56 +376,56 @@ public class DMLScript // (core compilation and execute) //////// - /** - * The running body of DMLScript execution. This method should be called after execution properties have been correctly set, - * and customized parameters have been put into _argVals - * - * @param dmlScriptStr DML script string - * @param fnameOptConfig configuration file - * @param argVals map of argument values - * @param allArgs arguments - * @param scriptType type of script (DML or PyDML) - * @throws IOException if IOException occurs - */ - private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs, ScriptType scriptType) - throws IOException - { - SCRIPT_TYPE = scriptType; - - //print basic time and environment info - printStartExecInfo( dmlScriptStr ); - - //Step 1: parse configuration files & write any configuration specific global variables - DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig); - ConfigurationManager.setGlobalConfig(dmlconf); - CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf); - ConfigurationManager.setGlobalConfig(cconf); - LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo()); - - setGlobalFlags(dmlconf); - Program rtprog = ScriptExecutorUtils.compileRuntimeProgram(dmlScriptStr, argVals, allArgs, - scriptType, dmlconf, SystemMLAPI.DMLScript); - List<GPUContext> gCtxs = ConfigurationManager.getDMLOptions().gpu ? GPUContextPool.getAllGPUContexts() : null; - - //double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext()); - //System.out.println("Estimated costs: "+costs); - - //Step 10: execute runtime program - ExecutionContext ec = null; - try { - ec = ScriptExecutorUtils.executeRuntimeProgram( - rtprog, dmlconf, ConfigurationManager.isStatistics() ? - ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0, - new LocalVariableMap(), null, SystemMLAPI.DMLScript, gCtxs); - } - finally { - if(ec != null && ec instanceof SparkExecutionContext) - ((SparkExecutionContext) ec).close(); - LOG.info("END DML run " + getDateTime() ); - //cleanup scratch_space and all working dirs - cleanupHadoopExecution( dmlconf ); - } - } + /** + * The running body of DMLScript execution. This method should be called after execution properties have been correctly set, + * and customized parameters have been put into _argVals + * + * @param dmlScriptStr DML script string + * @param fnameOptConfig configuration file + * @param argVals map of argument values + * @param allArgs arguments + * @param scriptType type of script (DML or PyDML) + * @throws IOException if IOException occurs + */ + private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs, ScriptType scriptType) + throws IOException + { + SCRIPT_TYPE = scriptType; + + //print basic time and environment info + printStartExecInfo( dmlScriptStr ); + + //Step 1: parse configuration files & write any configuration specific global variables + DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig); + ConfigurationManager.setGlobalConfig(dmlconf); + CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf); + ConfigurationManager.setGlobalConfig(cconf); + LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo()); + + setGlobalFlags(dmlconf); + Program rtprog = ScriptExecutorUtils.compileRuntimeProgram(dmlScriptStr, argVals, allArgs, + scriptType, dmlconf, SystemMLAPI.DMLScript); + List<GPUContext> gCtxs = ConfigurationManager.getDMLOptions().gpu ? GPUContextPool.getAllGPUContexts() : null; + + //double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext()); + //System.out.println("Estimated costs: "+costs); + + //Step 10: execute runtime program + ExecutionContext ec = null; + try { + ec = ScriptExecutorUtils.executeRuntimeProgram( + rtprog, ConfigurationManager.isStatistics() ? + ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0, + new LocalVariableMap(), null, SystemMLAPI.DMLScript, gCtxs); + } + finally { + if(ec != null && ec instanceof SparkExecutionContext) + ((SparkExecutionContext) ec).close(); + LOG.info("END DML run " + getDateTime() ); + //cleanup scratch_space and all working dirs + cleanupHadoopExecution( dmlconf ); + } + } /** * Sets the global flags in DMLScript based on user provided configuration @@ -547,8 +547,8 @@ public class DMLScript //determine security states boolean flagDiffUser = !( taskController.equals("org.apache.hadoop.mapred.LinuxTaskController") //runs map/reduce tasks as the current user - || localMode // run in the same JVM anyway - || groupNames.contains( ttGroupName) ); //user in task tracker group + || localMode // run in the same JVM anyway + || groupNames.contains( ttGroupName) ); //user in task tracker group boolean flagLocalFS = fsURI==null || fsURI.getScheme().equals("file"); boolean flagSecurity = perm.equals("yes"); http://git-wip-us.apache.org/repos/asf/systemml/blob/bf4ba16b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java index b1b5735..e2e76cb 100644 --- a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java +++ b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java @@ -116,7 +116,7 @@ public class ScriptExecutorUtils { boolean init) { DMLScript.SCRIPT_TYPE = scriptType; - Program rtprog = null; + Program rtprog; if (ConfigurationManager.isGPU() && !IS_JCUDA_AVAILABLE) throw new RuntimeException("Incorrect usage: Cannot use the GPU backend without JCuda libraries. Hint: Include systemml-*-extra.jar (compiled using mvn package -P distribution) into the classpath."); @@ -161,7 +161,7 @@ public class ScriptExecutorUtils { //init working directories (before usage by following compilation steps) if(api != SystemMLAPI.JMLC) - if ((api == SystemMLAPI.MLContext && init) || api != SystemMLAPI.MLContext) + if (api != SystemMLAPI.MLContext || init) DMLScript.initHadoopExecution( dmlconf ); @@ -222,8 +222,9 @@ public class ScriptExecutorUtils { ExplainCounts counts = Explain.countDistributedOperations(rtprog); Statistics.resetNoOfCompiledJobs( counts.numJobs ); //explain plan of program (hops or runtime) - if( DMLScript.EXPLAIN != ExplainType.NONE ) - System.out.println(Explain.display(prog, rtprog, DMLScript.EXPLAIN, counts)); + if( ConfigurationManager.getDMLOptions().explainType != ExplainType.NONE ) + System.out.println( + Explain.display(prog, rtprog, ConfigurationManager.getDMLOptions().explainType, counts)); Statistics.stopCompileTimer(); } @@ -232,9 +233,6 @@ public class ScriptExecutorUtils { // don't chain ParseException (for cleaner error output) throw pe; } - catch(IOException ex) { - throw new DMLException(ex); - } catch(Exception ex) { throw new DMLException(ex); } @@ -248,8 +246,6 @@ public class ScriptExecutorUtils { * * @param rtprog * runtime program - * @param dmlconf - * dml configuration * @param statisticsMaxHeavyHitters * maximum number of statistics to print * @param symbolTable @@ -262,7 +258,7 @@ public class ScriptExecutorUtils { * list of GPU contexts * @return execution context */ - public static ExecutionContext executeRuntimeProgram(Program rtprog, DMLConfig dmlconf, int statisticsMaxHeavyHitters, + public static ExecutionContext executeRuntimeProgram(Program rtprog, int statisticsMaxHeavyHitters, LocalVariableMap symbolTable, HashSet<String> outputVariables, SystemMLAPI api, List<GPUContext> gCtxs) { boolean exceptionThrown = false; @@ -299,7 +295,7 @@ public class ScriptExecutorUtils { if(outputVariables != null) { for(String outVar : outputVariables) { Data data = ec.getVariable(outVar); - if(data != null && data instanceof MatrixObject) { + if(data instanceof MatrixObject) { for(GPUContext gCtx : ec.getGPUContexts()) { GPUObject gpuObj = ((MatrixObject)data).getGPUObject(gCtx); if(gpuObj != null && gpuObj.isDirty()) { @@ -341,4 +337,4 @@ public class ScriptExecutorUtils { return ec; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/systemml/blob/bf4ba16b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java index 5224097..edbd5bf 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java +++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java @@ -32,7 +32,6 @@ import org.apache.sysml.api.ConfigurableAPI; import org.apache.sysml.api.DMLException; import org.apache.sysml.api.ScriptExecutorUtils; import org.apache.sysml.api.ScriptExecutorUtils.SystemMLAPI; -import org.apache.sysml.api.DMLScript; import org.apache.sysml.conf.CompilerConfig; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; @@ -47,8 +46,6 @@ import org.apache.sysml.runtime.controlprogram.LocalVariableMap; import org.apache.sysml.runtime.controlprogram.Program; import org.apache.sysml.runtime.controlprogram.caching.FrameObject; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; -import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; -import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory; import org.apache.sysml.runtime.instructions.cp.BooleanObject; import org.apache.sysml.runtime.instructions.cp.Data; import org.apache.sysml.runtime.instructions.cp.DoubleObject; @@ -100,6 +97,9 @@ public class PreparedScript implements ConfigurableAPI _inVarReuse = new HashMap<>(that._inVarReuse); _dmlconf = that._dmlconf; _cconf = that._cconf; + _isStatisticsEnabled = that._isStatisticsEnabled; + _gatherMemStats = that._gatherMemStats; + _gpuCtx = that._gpuCtx; } /** @@ -463,7 +463,7 @@ public class PreparedScript implements ConfigurableAPI //create and populate execution context ScriptExecutorUtils.executeRuntimeProgram( - _prog, _dmlconf, ConfigurationManager.isStatistics() ? + _prog, ConfigurationManager.isStatistics() ? ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0, _vars, _outVarnames, SystemMLAPI.JMLC, _gpuCtx); http://git-wip-us.apache.org/repos/asf/systemml/blob/bf4ba16b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java index 06778b3..7bda306 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java @@ -19,7 +19,6 @@ package org.apache.sysml.api.mlcontext; -import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -31,21 +30,15 @@ import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.api.ScriptExecutorUtils; import org.apache.sysml.api.ScriptExecutorUtils.SystemMLAPI; -import org.apache.sysml.api.jmlc.JMLCUtils; import org.apache.sysml.api.mlcontext.MLContext.ExecutionType; import org.apache.sysml.api.mlcontext.MLContext.ExplainLevel; import org.apache.sysml.conf.CompilerConfig; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.conf.DMLOptions; -import org.apache.sysml.hops.HopsException; import org.apache.sysml.hops.OptimizerUtils; -import org.apache.sysml.hops.rewrite.ProgramRewriter; -import org.apache.sysml.hops.rewrite.RewriteRemovePersistentReadWrite; -import org.apache.sysml.lops.LopsException; import org.apache.sysml.parser.DMLProgram; import org.apache.sysml.parser.DMLTranslator; -import org.apache.sysml.parser.LanguageException; import org.apache.sysml.parser.ParseException; import org.apache.sysml.parser.ParserFactory; import org.apache.sysml.parser.ParserWrapper; @@ -56,7 +49,6 @@ import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.instructions.gpu.context.GPUContext; import org.apache.sysml.runtime.instructions.gpu.context.GPUContextPool; import org.apache.sysml.utils.Explain; -import org.apache.sysml.utils.Explain.ExplainCounts; import org.apache.sysml.utils.Explain.ExplainType; import org.apache.sysml.utils.Statistics; @@ -102,7 +94,6 @@ public class ScriptExecutor { protected DMLConfig config; protected DMLProgram dmlProgram; - protected DMLTranslator dmlTranslator; protected Program runtimeProgram; protected ExecutionContext executionContext; protected Script script; @@ -139,80 +130,6 @@ public class ScriptExecutor { } /** - * Construct DAGs of high-level operators (HOPs) for each block of - * statements. - */ - protected void constructHops() { - try { - dmlTranslator.constructHops(dmlProgram); - } catch (LanguageException | ParseException e) { - throw new MLContextException("Exception occurred while constructing HOPS (high-level operators)", e); - } - } - - /** - * Apply static rewrites, perform intra-/inter-procedural analysis to - * propagate size information into functions, apply dynamic rewrites, and - * compute memory estimates for all HOPs. - */ - protected void rewriteHops() { - try { - dmlTranslator.rewriteHopsDAG(dmlProgram); - } catch (LanguageException | HopsException | ParseException | DMLRuntimeException e) { - throw new MLContextException("Exception occurred while rewriting HOPS (high-level operators)", e); - } - } - - /** - * Output a description of the program to standard output. - */ - protected void showExplanation() { - if (!explain) - return; - - try { - ExplainType explainType = (explainLevel != null) ? explainLevel.getExplainType() : ExplainType.RUNTIME; - System.out.println(Explain.display(dmlProgram, runtimeProgram, explainType, null)); - } catch (Exception e) { - throw new MLContextException("Exception occurred while explaining dml program", e); - } - } - - /** - * Construct DAGs of low-level operators (LOPs) based on the DAGs of - * high-level operators (HOPs). - */ - protected void constructLops() { - try { - dmlTranslator.constructLops(dmlProgram); - } catch (ParseException | LanguageException | HopsException | LopsException e) { - throw new MLContextException("Exception occurred while constructing LOPS (low-level operators)", e); - } - } - - /** - * Create runtime program. For each namespace, translate function statement - * blocks into function program blocks and add these to the runtime program. - * For each top-level block, add the program block to the runtime program. - */ - protected void generateRuntimeProgram() { - try { - runtimeProgram = dmlTranslator.getRuntimeProgram(dmlProgram, config); - } catch (LanguageException | DMLRuntimeException | LopsException | IOException | HopsException e) { - throw new MLContextException("Exception occurred while generating runtime program", e); - } - } - - /** - * Count the number of compiled MR Jobs/Spark Instructions in the runtime - * program and set this value in the statistics. - */ - protected void countCompiledMRJobsAndSparkInstructions() { - ExplainCounts counts = Explain.countDistributedOperations(runtimeProgram); - Statistics.resetNoOfCompiledJobs(counts.numJobs); - } - - /** * Set the global flags (for example: statistics, gpu, etc). */ protected void setGlobalFlags() { @@ -255,25 +172,6 @@ public class ScriptExecutor { /** * Compile a DML or PYDML script. This will help analysis of DML programs * that have dynamic recompilation flag set to false without actually executing it. - * - * This is broken down into the following - * primary methods: - * - * <ol> - * <li>{@link #setup(Script)}</li> - * <li>{@link #parseScript()}</li> - * <li>{@link #liveVariableAnalysis()}</li> - * <li>{@link #validateScript()}</li> - * <li>{@link #constructHops()}</li> - * <li>{@link #rewriteHops()}</li> - * <li>{@link #rewritePersistentReadsAndWrites()}</li> - * <li>{@link #constructLops()}</li> - * <li>{@link #generateRuntimeProgram()}</li> - * <li>{@link #showExplanation()}</li> - * <li>{@link #countCompiledMRJobsAndSparkInstructions()}</li> - * <li>{@link #initializeCachingAndScratchSpace()}</li> - * <li>{@link #cleanupRuntimeProgram()}</li> - * </ol> * * @param script * the DML or PYDML script to compile @@ -321,8 +219,8 @@ public class ScriptExecutor { .convertInputParametersForParser(script.getInputParameters(), script.getScriptType()); Explain.ExplainType explainType = Explain.ExplainType.NONE; - if(explain && explainLevel != null) { - explainType = explainLevel.getExplainType(); + if(explain) { + explainType = (explainLevel == null) ? Explain.ExplainType.RUNTIME : explainLevel.getExplainType(); } RUNTIME_PLATFORM rtplatform = DMLOptions.defaultOptions.execMode; if(executionType != null) { @@ -332,14 +230,14 @@ public class ScriptExecutor { statistics, statisticsMaxHeavyHitters, false, explainType, rtplatform, gpu, forceGPU, script.getScriptType(), DMLScript.DML_FILE_PATH_ANTLR_PARSER, script.getScriptExecutionString())); - + // main steps in script execution compile(script); try { - executionContext = ScriptExecutorUtils.executeRuntimeProgram(getRuntimeProgram(), getConfig(), + executionContext = ScriptExecutorUtils.executeRuntimeProgram(getRuntimeProgram(), statistics ? statisticsMaxHeavyHitters : 0, script.getSymbolTable(), - new HashSet<String>(getScript().getOutputVariables()), SystemMLAPI.MLContext, gCtxs); + new HashSet<>(getScript().getOutputVariables()), SystemMLAPI.MLContext, gCtxs); } catch (DMLRuntimeException e) { throw new MLContextException("Exception occurred while executing runtime program", e); } finally { @@ -411,41 +309,6 @@ public class ScriptExecutor { } /** - * If {@code maintainSymbolTable} is true, delete all 'remove variable' - * instructions so as to maintain the values in the symbol table, which are - * useful when working interactively in an environment such as the Spark - * Shell. Otherwise, only delete 'remove variable' instructions for - * registered outputs. - */ - protected void cleanupRuntimeProgram() { - if (maintainSymbolTable) { - MLContextUtil.deleteRemoveVariableInstructions(runtimeProgram); - } else { - JMLCUtils.cleanupRuntimeProgram(runtimeProgram, (script.getOutputVariables() == null) ? new String[0] - : script.getOutputVariables().toArray(new String[0])); - } - } - - /** - * Check security, create scratch space, cleanup working directories, - * initialize caching, and reset statistics. - */ - protected void initializeCachingAndScratchSpace() { - if (!init) - return; - - try { - DMLScript.initHadoopExecution(config); - } catch (ParseException e) { - throw new MLContextException("Exception occurred initializing caching and scratch space", e); - } catch (DMLRuntimeException e) { - throw new MLContextException("Exception occurred initializing caching and scratch space", e); - } catch (IOException e) { - throw new MLContextException("Exception occurred initializing caching and scratch space", e); - } - } - - /** * Parse the script into an ANTLR parse tree, and convert this parse tree * into a SystemML program. Parsing includes lexical/syntactic analysis. */ @@ -461,29 +324,6 @@ public class ScriptExecutor { } /** - * Replace persistent reads and writes with transient reads and writes in - * the symbol table. - */ - protected void rewritePersistentReadsAndWrites() { - LocalVariableMap symbolTable = script.getSymbolTable(); - if (symbolTable != null) { - String[] inputs = (script.getInputVariables() == null) ? new String[0] - : script.getInputVariables().toArray(new String[0]); - String[] outputs = (script.getOutputVariables() == null) ? new String[0] - : script.getOutputVariables().toArray(new String[0]); - RewriteRemovePersistentReadWrite rewrite = new RewriteRemovePersistentReadWrite(inputs, outputs, - script.getSymbolTable()); - ProgramRewriter programRewriter = new ProgramRewriter(rewrite); - try { - programRewriter.rewriteProgramHopDAGs(dmlProgram); - } catch (LanguageException | HopsException e) { - throw new MLContextException("Exception occurred while rewriting persistent reads and writes", e); - } - } - - } - - /** * Set the SystemML configuration properties. * * @param config @@ -495,50 +335,6 @@ public class ScriptExecutor { } /** - * Liveness analysis is performed on the program, obtaining sets of live-in - * and live-out variables by forward and backward passes over the program. - */ - protected void liveVariableAnalysis() { - try { - dmlTranslator = new DMLTranslator(dmlProgram); - dmlTranslator.liveVariableAnalysis(dmlProgram); - } catch (DMLRuntimeException e) { - throw new MLContextException("Exception occurred during live variable analysis", e); - } catch (LanguageException e) { - throw new MLContextException("Exception occurred during live variable analysis", e); - } - } - - /** - * Semantically validate the program's expressions, statements, and - * statement blocks in a single recursive pass over the program. Constant - * and size propagation occurs during this step. - */ - protected void validateScript() { - try { - dmlTranslator.validateParseTree(dmlProgram); - } catch (LanguageException | ParseException e) { - throw new MLContextException("Exception occurred while validating script", e); - } - } - - /** - * Check that the Script object has a type (DML or PYDML) and a string - * representing the content of the Script. - */ - protected void checkScriptHasTypeAndString() { - if (script == null) { - throw new MLContextException("Script is null"); - } else if (script.getScriptType() == null) { - throw new MLContextException("ScriptType (DML or PYDML) needs to be specified"); - } else if (script.getScriptString() == null) { - throw new MLContextException("Script string is null"); - } else if (StringUtils.isBlank(script.getScriptString())) { - throw new MLContextException("Script string is blank"); - } - } - - /** * Obtain the program * * @return the program @@ -548,15 +344,6 @@ public class ScriptExecutor { } /** - * Obtain the translator - * - * @return the translator - */ - public DMLTranslator getDmlTranslator() { - return dmlTranslator; - } - - /** * Obtain the runtime program * * @return the runtime program @@ -591,9 +378,7 @@ public class ScriptExecutor { * {@code true} if explanation should be output, {@code false} * otherwise */ - public void setExplain(boolean explain) { - this.explain = explain; - } + public void setExplain(boolean explain) { this.explain = explain; } /** * Whether or not statistics about the DML/PYDML program should be output to @@ -664,8 +449,7 @@ public class ScriptExecutor { if (explainLevel == null) { DMLScript.EXPLAIN = ExplainType.NONE; } else { - ExplainType explainType = explainLevel.getExplainType(); - DMLScript.EXPLAIN = explainType; + DMLScript.EXPLAIN = explainLevel.getExplainType(); } } @@ -717,4 +501,4 @@ public class ScriptExecutor { ConfigurationManager.getDMLOptions().setExecutionMode(executionType.getRuntimePlatform()); this.executionType = executionType; } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/systemml/blob/bf4ba16b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java index 08d929e..c10b3a5 100644 --- a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java +++ b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java @@ -37,10 +37,10 @@ import org.apache.sysml.utils.lite.LiteCheck; public class ConfigurationManager { /** Global cached job conf for read-only operations */ - private static JobConf _rJob = null; + private static JobConf _rJob; /** Global DML configuration (read or defaults) */ - private static DMLConfig _dmlconf = null; + private static DMLConfig _dmlconf; /** Local DML configuration for thread-local config updates */ private static ThreadLocalDMLConfig _ldmlconf = new ThreadLocalDMLConfig(); @@ -52,7 +52,7 @@ public class ConfigurationManager private static ThreadLocalDMLOptions _ldmlOptions = new ThreadLocalDMLOptions(); /** Global compiler configuration (defaults) */ - private static CompilerConfig _cconf = null; + private static CompilerConfig _cconf; /** Local compiler configuration for thead-local config updates */ private static ThreadLocalCompilerConfig _lcconf = new ThreadLocalCompilerConfig(); @@ -208,7 +208,7 @@ public class ConfigurationManager */ public static boolean getCompilerConfigFlag(ConfigType key) { CompilerConfig cconf = getCompilerConfig(); - return (cconf!=null) ? cconf.getBool(key) : false; + return (cconf!=null) && cconf.getBool(key); } ///////////////////////////////////// http://git-wip-us.apache.org/repos/asf/systemml/blob/bf4ba16b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java index 220f3e5..58a8694 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java @@ -26,7 +26,6 @@ import java.util.Map.Entry; import java.util.Set; import java.util.StringTokenizer; -import org.apache.sysml.api.DMLScript; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.runtime.controlprogram.caching.CacheableData; import org.apache.sysml.runtime.util.ProgramConverter; http://git-wip-us.apache.org/repos/asf/systemml/blob/bf4ba16b/src/test/java/org/apache/sysml/test/gpu/JMLCTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/JMLCTests.java b/src/test/java/org/apache/sysml/test/gpu/JMLCTests.java index d77d794..be02976 100644 --- a/src/test/java/org/apache/sysml/test/gpu/JMLCTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/JMLCTests.java @@ -28,99 +28,97 @@ import org.apache.sysml.api.jmlc.PreparedScript; public class JMLCTests extends GPUTests { - static class ScriptContainer { - String dml; - String[] inputVarNames; - } - - @Test - public void testJMLC() { - try { - Connection conn = new Connection(); - - int numMatrices = 10; - int matrixNumRows = 100; - int numScriptInvocations = 10; - - ScriptContainer SC = generateDMLScript(numMatrices); - - PreparedScript script = conn.prepareScript( - SC.dml, SC.inputVarNames, new String[] { "Z" }, true, true, 0); - - // execute the script without pinning input matrices between invocations - executeDMLScript(script, numScriptInvocations, matrixNumRows, numMatrices, false); - - // execute the script while pinning input matrices between invocations - executeDMLScript(script, numScriptInvocations, matrixNumRows, numMatrices, true); - } catch (Exception e) { - Assert.fail("An unexpected exception occurred: " + e.getMessage()); - } - } - - // Generates a simple synthetic DML script which multiplies a sequence of square matrices. - // I.e. Z = X %*% W1 %*% W2 %*% W3 ... - // numMatrices determines the number of matrices in the sequences. The size of the matrices can be set - // in executeDMLScript - static ScriptContainer generateDMLScript(int numMatrices) { - ScriptContainer SC = new ScriptContainer(); - String[] inputVarNames = new String[numMatrices + 1]; - inputVarNames[0] = "x"; - - StringBuilder dml = new StringBuilder("x = read(\"/tmp/X.mtx\", rows=-1, cols=-1)\n"); - for (int ix=0; ix<numMatrices; ix++) - { - String name = "W" + ix; - inputVarNames[ix+1] = name; - dml.append(name + " = read(\"/tmp/" + name + ".mtx\", rows=-1, cols=-1)\n"); - } - - dml.append("Z = x %*% W0\n"); - for (int ix=1; ix<numMatrices; ix++) - { - dml.append("Z = Z %*% W" + ix + "\n"); - } - - dml.append("while (-1 > 1)\n print(as.scalar(Z[1,1]))\n"); - - SC.dml = dml.toString(); - SC.inputVarNames = inputVarNames; - - return SC; - } - - // Executes a PreparedScript generated by generateDMLScript. The parameter n determines the - // number of times the script is invoked. The parameter rows controls the shape of the matrices. - // Set this parameter larger to use more memory. The parameter numMatrices must be set to the same value as - // in generateDMLScript. The parameter pinWeights controls whether weight matrices should be - // pinned in memory between script invocations. - static void executeDMLScript(PreparedScript script, int n, int rows, int numMatrices, boolean pinWeights) { - for (int ix=0; ix<numMatrices; ix++) - script.setMatrix("W" + ix, randomMatrix(rows, rows, 0.0,1.0, 1.0), pinWeights); - - for (int ix=0; ix<n; ix++) - { - script.setMatrix("x", randomMatrix(rows, rows, 0.0, 1.0, 1.0), false); - script.executeScript(); - if (!pinWeights) - for (int iy=0; iy<numMatrices; iy++) - script.setMatrix( - "W" + iy, randomMatrix(rows, rows, 0.0,1.0, 1.0), false); - } - } - - static double[][] randomMatrix( - int rows, int cols, double min, double max, double sparsity) { - double[][] matrix = new double[rows][cols]; - Random random = new Random(System.currentTimeMillis()); - for (int i = 0; i < rows; i++) { - for (int j = 0; j < cols; j++) { - if (random.nextDouble() > sparsity) { - continue; - } - matrix[i][j] = (random.nextDouble() * (max - min) + min); - } - } - return matrix; - } + static class ScriptContainer { + String dml; + String[] inputVarNames; + } + + @Test + public void testJMLC() { + try { + Connection conn = new Connection(); + + int numMatrices = 10; + int matrixNumRows = 100; + int numScriptInvocations = 10; + + ScriptContainer SC = generateDMLScript(numMatrices); + + PreparedScript script = conn.prepareScript( + SC.dml, SC.inputVarNames, new String[] { "Z" }, true, true, 0); + + // execute the script without pinning input matrices between invocations + executeDMLScript(script, numScriptInvocations, matrixNumRows, numMatrices, false); + + // execute the script while pinning input matrices between invocations + executeDMLScript(script, numScriptInvocations, matrixNumRows, numMatrices, true); + } catch (Exception e) { + Assert.fail("An unexpected exception occurred: " + e.getMessage()); + } + } + + // Generates a simple synthetic DML script which multiplies a sequence of square matrices. + // I.e. Z = X %*% W1 %*% W2 %*% W3 ... + // numMatrices determines the number of matrices in the sequences. The size of the matrices can be set + // in executeDMLScript + private static ScriptContainer generateDMLScript(int numMatrices) { + ScriptContainer SC = new ScriptContainer(); + String[] inputVarNames = new String[numMatrices + 1]; + inputVarNames[0] = "x"; + + StringBuilder dml = new StringBuilder("x = read(\"/tmp/X.mtx\", rows=-1, cols=-1)\n"); + for (int ix=0; ix<numMatrices; ix++) + { + String name = "W" + ix; + inputVarNames[ix+1] = name; + String readCmd = name + " = read(\"/tmp/" + name + ".mtx\", rows=-1, cols=-1)\n"; + dml.append(readCmd); + } + + dml.append("Z = x %*% W0\n"); + for (int ix=1; ix<numMatrices; ix++) + { + String multiplyCmd = "Z = Z %*% W" + ix + "\n"; + dml.append(multiplyCmd); + } + + dml.append("while (-1 > 1)\n print(as.scalar(Z[1,1]))\n"); + + SC.dml = dml.toString(); + SC.inputVarNames = inputVarNames; + + return SC; + } + + // Executes a PreparedScript generated by generateDMLScript. The parameter n determines the + // number of times the script is invoked. The parameter rows controls the shape of the matrices. + // Set this parameter larger to use more memory. The parameter numMatrices must be set to the same value as + // in generateDMLScript. The parameter pinWeights controls whether weight matrices should be + // pinned in memory between script invocations. + private static void executeDMLScript(PreparedScript script, int n, int rows, int numMatrices, boolean pinWeights) { + for (int ix=0; ix<numMatrices; ix++) + script.setMatrix("W" + ix, randomMatrix(rows, rows), pinWeights); + + for (int ix=0; ix<n; ix++) + { + script.setMatrix("x", randomMatrix(rows, rows), false); + script.executeScript(); + if (!pinWeights) + for (int iy=0; iy<numMatrices; iy++) + script.setMatrix( + "W" + iy, randomMatrix(rows, rows), false); + } + } + + private static double[][] randomMatrix(int rows, int cols) { + double[][] matrix = new double[rows][cols]; + Random random = new Random(System.currentTimeMillis()); + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + matrix[i][j] = random.nextDouble(); + } + } + return matrix; + } } http://git-wip-us.apache.org/repos/asf/systemml/blob/bf4ba16b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java index 345ea7a..f0ac1c1 100644 --- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java +++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java @@ -413,10 +413,7 @@ public class MLContextTest extends MLContextTestBase { setExpectedStdOut(testString); Script script = new Script("print('" + testString + "');", org.apache.sysml.api.mlcontext.ScriptType.DML); - ScriptExecutor scriptExecutor = new ScriptExecutor() { - @Override - protected void showExplanation() {} - }; + ScriptExecutor scriptExecutor = new ScriptExecutor(); ml.execute(script, scriptExecutor); } @@ -427,10 +424,7 @@ public class MLContextTest extends MLContextTestBase { setExpectedStdOut(testString); Script script = new Script("print('" + testString + "')", org.apache.sysml.api.mlcontext.ScriptType.PYDML); - ScriptExecutor scriptExecutor = new ScriptExecutor() { - @Override - protected void showExplanation() {} - }; + ScriptExecutor scriptExecutor = new ScriptExecutor(); ml.execute(script, scriptExecutor); }
