http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/api/DMLScript.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java index c04c321..175688a 100644 --- a/src/main/java/org/apache/sysml/api/DMLScript.java +++ b/src/main/java/org/apache/sysml/api/DMLScript.java @@ -19,22 +19,15 @@ package org.apache.sysml.api; -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URI; -import java.text.DateFormat; -import java.text.SimpleDateFormat; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Scanner; - +import org.apache.commons.cli.AlreadySelectedException; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -46,7 +39,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.log4j.Level; import org.apache.log4j.Logger; -import org.apache.sysml.api.mlcontext.ScriptType; import org.apache.sysml.conf.CompilerConfig; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; @@ -73,12 +65,12 @@ import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics; import org.apache.sysml.runtime.controlprogram.caching.CacheableData; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory; -import org.apache.sysml.runtime.instructions.gpu.context.GPUContext; -import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; import org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter; import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler; +import org.apache.sysml.runtime.instructions.gpu.context.GPUContext; +import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.matrix.CleanupMR; import org.apache.sysml.runtime.matrix.data.LibMatrixDNN; import org.apache.sysml.runtime.matrix.mapred.MRConfigurationNames; @@ -93,6 +85,22 @@ import org.apache.sysml.utils.Statistics; import org.apache.sysml.yarn.DMLAppMasterUtils; import org.apache.sysml.yarn.DMLYarnClientProxy; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Scanner; + public class DMLScript { @@ -103,66 +111,77 @@ public class DMLScript HYBRID_SPARK, // execute matrix operations in CP or Spark SPARK // execute matrix operations in Spark } - - public static RUNTIME_PLATFORM rtplatform = OptimizerUtils.getDefaultExecutionMode(); - public static boolean STATISTICS = false; //default statistics - public static int STATISTICS_COUNT = 10; //default statistics maximum heavy hitter count - public static boolean ENABLE_DEBUG_MODE = false; //default debug mode - public static boolean USE_LOCAL_SPARK_CONFIG = false; //set default local spark configuration - used for local testing - public static String DML_FILE_PATH_ANTLR_PARSER = null; - public static ExplainType EXPLAIN = ExplainType.NONE; //default explain + + /** + * Set of DMLOptions that can be set through the command line + * and {@link org.apache.sysml.api.mlcontext.MLContext} + * The values have been initialized with the default values + * Despite there being a DML and PyDML, this class is named DMLOptions + * to keep it consistent with {@link DMLScript} and {@link DMLOptions} + */ + public static class DMLOptions { + public Map<String, String> argVals = new HashMap<>(); // Arguments map containing either named arguments or arguments by position for a DML program + public String configFile = null; // Path to config file if default config and default config is to be overriden + public boolean clean = false; // Whether to clean up all SystemML working directories (FS, DFS) + public boolean stats = false; // Whether to record and print the statistics + public int statsCount = 10; // Default statistics count + public Explain.ExplainType explainType = Explain.ExplainType.NONE; // Whether to print the "Explain" and if so, what type + public DMLScript.RUNTIME_PLATFORM execMode = OptimizerUtils.getDefaultExecutionMode(); // Execution mode standalone, MR, Spark or a hybrid + public boolean gpu = false; // Whether to use the GPU + public boolean forceGPU = false; // Whether to ignore memory & estimates and always use the GPU + public boolean debug = false; // to go into debug mode to be able to step through a program + public ScriptType scriptType = ScriptType.DML; // whether the script is a DML or PyDML script + public String filePath = null; // path to script + public String script = null; // the script itself + public boolean help = false; // whether to print the usage option + + public final static DMLOptions defaultOptions = new DMLOptions(); + + @Override + public String toString() { + return "DMLOptions{" + + "argVals=" + argVals + + ", configFile='" + configFile + '\'' + + ", clean=" + clean + + ", stats=" + stats + + ", statsCount=" + statsCount + + ", explainType=" + explainType + + ", execMode=" + execMode + + ", gpu=" + gpu + + ", forceGPU=" + forceGPU + + ", debug=" + debug + + ", scriptType=" + scriptType + + ", filePath='" + filePath + '\'' + + ", script='" + script + '\'' + + ", help=" + help + + '}'; + } + } + + public static RUNTIME_PLATFORM rtplatform = DMLOptions.defaultOptions.execMode; // the execution mode + public static boolean STATISTICS = DMLOptions.defaultOptions.stats; // whether to print statistics + public static int STATISTICS_COUNT = DMLOptions.defaultOptions.statsCount; // statistics maximum heavy hitter count + public static boolean ENABLE_DEBUG_MODE = DMLOptions.defaultOptions.debug; // debug mode + public static ExplainType EXPLAIN = DMLOptions.defaultOptions.explainType; // explain type + public static String DML_FILE_PATH_ANTLR_PARSER = DMLOptions.defaultOptions.filePath; // filename of dml/pydml script + /** * Global variable indicating the script type (DML or PYDML). Can be used * for DML/PYDML-specific tasks, such as outputting booleans in the correct * case (TRUE/FALSE for DML and True/False for PYDML). */ - public static ScriptType SCRIPT_TYPE = ScriptType.DML; - - public static boolean USE_ACCELERATOR = false; - public static boolean FORCE_ACCELERATOR = false; - - // flag that indicates whether or not to suppress any prints to stdout - public static boolean _suppressPrint2Stdout = false; - - public static String _uuid = IDHandler.createDistributedUniqueID(); + public static ScriptType SCRIPT_TYPE = DMLOptions.defaultOptions.scriptType; + public static boolean USE_ACCELERATOR = DMLOptions.defaultOptions.gpu; + public static boolean FORCE_ACCELERATOR = DMLOptions.defaultOptions.forceGPU; + + + public static boolean _suppressPrint2Stdout = false; // flag that indicates whether or not to suppress any prints to stdout + public static boolean USE_LOCAL_SPARK_CONFIG = false; //set default local spark configuration - used for local testing public static boolean _activeAM = false; - + + public static String _uuid = IDHandler.createDistributedUniqueID(); private static final Log LOG = LogFactory.getLog(DMLScript.class.getName()); - public static String USAGE = - "Usage is " + DMLScript.class.getCanonicalName() + " -f <filename>" - //+ " (-exec <mode>)?" + " (-explain <type>)?" + " (-stats)?" + " (-clean)?" + " (-config=<config_filename>)? - + " [-options] ([-args | -nvargs] <args-list>)? \n" - + " -f: <filename> will be interpreted as a filename path (if <filename> is prefixed\n" - + " with hdfs or gpfs it is read from DFS, otherwise from local file system)\n" - //undocumented feature in beta 08/2014 release - //+ " -s: <filename> will be interpreted as a DML script string \n" - + " -python: (optional) parses Python-like DML\n" - + " -debug: (optional) run in debug mode\n" - + " -gpu: <flags> (optional) use acceleration whenever possible. Current version only supports CUDA.\n" - + " Supported <flags> for this mode is force=(true|false)\n" - // Later add optional flags to indicate optimizations turned on or off. Currently they are turned off. - //+ " -debug: <flags> (optional) run in debug mode\n" - //+ " Optional <flags> that is supported for this mode is optimize=(on|off)\n" - + " -exec: <mode> (optional) execution mode (hadoop, singlenode, [hybrid], hybrid_spark)\n" - + " -explain: <type> (optional) explain plan (hops, [runtime], recompile_hops, recompile_runtime)\n" - + " -stats: <count> (optional) monitor and report caching/recompilation statistics, default heavy hitter count is 10\n" - + " -clean: (optional) cleanup all SystemML working directories (FS, DFS).\n" - + " All other flags are ignored in this mode. \n" - + " -config: (optional) use config file <config_filename> (default: use parameter\n" - + " values in default SystemML-config.xml config file; if <config_filename> is\n" - + " prefixed with hdfs or gpfs it is read from DFS, otherwise from local file system)\n" - + " -args: (optional) parameterize DML script with contents of [args list], ALL args\n" - + " after -args flag, each argument must be an unnamed-argument, where 1st value\n" - + " after -args will replace $1 in DML script, 2nd value will replace $2, etc.\n" - + " -nvargs: (optional) parameterize DML script with contents of [args list], ALL args\n" - + " after -nvargs flag, each argument must be be named-argument of form argName=argValue,\n" - + " where value will replace $argName in DML script, argName must be a valid DML variable\n" - + " name (start with letter, contain only letters, numbers, or underscores).\n" - + " <args-list>: (optional) args to DML script \n" - + " -? | -help: (optional) show this help message \n"; - - /////////////////////////////// // public external interface //////// @@ -193,21 +212,19 @@ public class DMLScript public static boolean isActiveAM(){ return _activeAM; } - - + /** - * Default DML script invocation (e.g., via 'hadoop jar SystemML.jar -f Test.dml') - * + * * @param args command-line arguments * @throws IOException if an IOException occurs * @throws DMLException if a DMLException occurs */ - public static void main(String[] args) + public static void main(String[] args) throws IOException, DMLException { Configuration conf = new Configuration(ConfigurationManager.getCachedJobConf()); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); - + try { DMLScript.executeScript(conf, otherArgs); } catch (ParseException pe) { @@ -216,7 +233,200 @@ public class DMLScript // In case of DMLScriptException, simply print the error message. System.err.println(e.getMessage()); } - } + } + + /** + * Parses command line arguments to create a {@link DMLOptions} instance with the correct options + * @param args arguments from the command line + * @param options an {@link Options} instance containing the options that need to be parsed + * @return an instance of {@link Options} that contain the correct {@link Option}s. + * @throws org.apache.commons.cli.ParseException if there is an incorrect option specified in the CLI + */ + public static DMLOptions parseCLArguments(String[] args, Options options) throws org.apache.commons.cli.ParseException { + + CommandLineParser clParser = new PosixParser(); + CommandLine line = clParser.parse(options, args); + + DMLOptions dmlOptions = new DMLOptions(); + dmlOptions.help = line.hasOption("help"); + dmlOptions.scriptType = line.hasOption("python") ? ScriptType.PYDML : ScriptType.DML; + dmlOptions.debug = line.hasOption("debug"); + dmlOptions.gpu = line.hasOption("gpu"); + if (dmlOptions.gpu) { + String force = line.getOptionValue("gpu"); + if (force != null) { + if (force.equalsIgnoreCase("force")) { + dmlOptions.forceGPU = true; + } else { + throw new org.apache.commons.cli.ParseException("Invalid argument specified for -gpu option"); + } + } + } + if (line.hasOption("exec")){ + String execMode = line.getOptionValue("exec"); + if (execMode != null){ + if (execMode.equalsIgnoreCase("hadoop")) dmlOptions.execMode = RUNTIME_PLATFORM.HADOOP; + else if (execMode.equalsIgnoreCase("singlenode")) dmlOptions.execMode = RUNTIME_PLATFORM.SINGLE_NODE; + else if (execMode.equalsIgnoreCase("hybrid")) dmlOptions.execMode = RUNTIME_PLATFORM.HYBRID; + else if (execMode.equalsIgnoreCase("hybrid_spark")) dmlOptions.execMode = RUNTIME_PLATFORM.HYBRID_SPARK; + else if (execMode.equalsIgnoreCase("spark")) dmlOptions.execMode = RUNTIME_PLATFORM.SPARK; + else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -exec option, must be one of [hadoop, singlenode, hybrid, hybrid_spark, spark]"); + } + } + if (line.hasOption("explain")) { + dmlOptions.explainType = ExplainType.RUNTIME; + String explainType = line.getOptionValue("explain"); + if (explainType != null){ + if (explainType.equalsIgnoreCase("hops")) dmlOptions.explainType = ExplainType.HOPS; + else if (explainType.equalsIgnoreCase("runtime")) dmlOptions.explainType = ExplainType.RUNTIME; + else if (explainType.equalsIgnoreCase("recompile_hops")) dmlOptions.explainType = ExplainType.RECOMPILE_HOPS; + else if (explainType.equalsIgnoreCase("recompile_runtime")) dmlOptions.explainType = ExplainType.RECOMPILE_RUNTIME; + else throw new org.apache.commons.cli.ParseException("Invalid argument specified for -hops option, must be one of [hops, runtime, recompile_hops, recompile_runtime]"); + } + } + dmlOptions.stats = line.hasOption("stats"); + if (dmlOptions.stats){ + String statsCount = line.getOptionValue("stats"); + if (statsCount != null) { + try { + dmlOptions.statsCount = Integer.parseInt(statsCount); + } catch (NumberFormatException e) { + throw new org.apache.commons.cli.ParseException("Invalid argument specified for -stats option, must be a valid integer"); + } + } + } + + dmlOptions.clean = line.hasOption("clean"); + + if (line.hasOption("config")){ + dmlOptions.configFile = line.getOptionValue("config"); + } + + if (line.hasOption("f")){ + dmlOptions.filePath = line.getOptionValue("f"); + } + + if (line.hasOption("s")){ + dmlOptions.script = line.getOptionValue("s"); + } + + // Positional arguments map is created as ("$1", "a"), ("$2", 123), .... + if (line.hasOption("args")){ + String[] argValues = line.getOptionValues("args"); + int k=1; + for (String str : argValues){ + if (!str.isEmpty()) { + dmlOptions.argVals.put("$" + k, str); + k++; + } + } + } + + // Named arguments map is created as ("$K, 123), ("$X", "X.csv"), .... + if (line.hasOption("nvargs")){ + String varNameRegex = "^[a-zA-Z]([a-zA-Z0-9_])*$"; + String[] nvargValues = line.getOptionValues("nvargs"); + for (String str : nvargValues){ + if (!str.isEmpty()){ + String[] kv = str.split("="); + if (kv.length != 2){ + throw new org.apache.commons.cli.ParseException("Invalid argument specified for -nvargs option, must be a list of space separated K=V pairs, where K is a valid name of a variable in the DML/PyDML program"); + } + if (!kv[0].matches(varNameRegex)) { + throw new org.apache.commons.cli.ParseException("Invalid argument specified for -nvargs option, " + kv[0] + " does not seem like a valid variable name in DML. Valid variable names in DML start with upper-case or lower-case letter, and contain only letters, digits, or underscores"); + } + dmlOptions.argVals.put("$" + kv[0], kv[1]); + } + } + } + + return dmlOptions; + + } + + /** + * Creates an {@link Options} instance for the command line parameters + * As of SystemML 0.13, Apache Commons CLI 1.2 is transitively in the classpath + * However the most recent version of Apache Commons CLI is 1.4 + * Creating CLI options is done using Static methods. This obviously makes it + * thread unsafe. Instead of {@link OptionBuilder}, CLI 1.4 uses Option.Builder which + * has non-static methods. + * @return an appropriate instance of {@link Options} + */ + @SuppressWarnings("static-access") + public static Options createCLIOptions() { + Options options = new Options(); + Option nvargsOpt = OptionBuilder.withArgName("key=value") + .withDescription("parameterizes DML script with named parameters of the form <key=value>; <key> should be a valid identifier in DML/PyDML") + .hasArgs() + .create("nvargs"); + Option argsOpt = OptionBuilder.withArgName("argN") + .withDescription("specifies positional parameters; first value will replace $1 in DML program; $2 will replace 2nd and so on") + .hasArgs() + .create("args"); + Option configOpt = OptionBuilder.withArgName("filename") + .withDescription("uses a given configuration file (can be on local/hdfs/gpfs; default values in SystemML-config.xml") + .hasArg() + .create("config"); + Option cleanOpt = OptionBuilder.withDescription("cleans up all SystemML working directories (FS, DFS); all other flags are ignored in this mode. \n") + .create("clean"); + Option statsOpt = OptionBuilder.withArgName("count") + .withDescription("monitors and reports caching/recompilation statistics; heavy hitter <count> is 10 unless overridden; default off") + .hasOptionalArg() + .create("stats"); + Option explainOpt = OptionBuilder.withArgName("level") + .withDescription("explains plan levels; can be 'hops' / 'runtime'[default] / 'recompile_hops' / 'recompile_runtime'") + .hasOptionalArg() + .create("explain"); + Option execOpt = OptionBuilder.withArgName("mode") + .withDescription("sets execution mode; can be 'hadoop' / 'singlenode' / 'hybrid'[default] / 'hybrid_spark' / 'spark'") + .hasArg() + .create("exec"); + Option gpuOpt = OptionBuilder.withArgName("force") + .withDescription("uses CUDA instructions when reasonable; set <force> option to skip conservative memory estimates and use GPU wherever possible; default off") + .hasOptionalArg() + .create("gpu"); + Option debugOpt = OptionBuilder.withDescription("runs in debug mode; default off") + .create("debug"); + Option pythonOpt = OptionBuilder.withDescription("parses Python-like DML") + .create("python"); + Option fileOpt = OptionBuilder.withArgName("filename") + .withDescription("specifies dml/pydml file to execute; path can be local/hdfs/gpfs (prefixed with appropriate URI)") + .isRequired() + .hasArg() + .create("f"); + Option scriptOpt = OptionBuilder.withArgName("script_contents") + .withDescription("specified script string to execute directly") + .isRequired() + .hasArg() + .create("s"); + Option helpOpt = OptionBuilder.withDescription("shows usage message") + .create("help"); + + OptionGroup fileOrScriptOpt = new OptionGroup(); + // Either a clean(-clean), a file(-f), a script(-s) or help(-help) needs to be specified + fileOrScriptOpt.addOption(scriptOpt); + fileOrScriptOpt.addOption(fileOpt); + fileOrScriptOpt.addOption(cleanOpt); + fileOrScriptOpt.addOption(helpOpt); + fileOrScriptOpt.setRequired(true); + + OptionGroup argsOrNVArgsOpt = new OptionGroup(); + argsOrNVArgsOpt.addOption(nvargsOpt).addOption(argsOpt); // Either -args or -nvargs + + options.addOption(configOpt); + options.addOption(cleanOpt); + options.addOption(statsOpt); + options.addOption(explainOpt); + options.addOption(execOpt); + options.addOption(gpuOpt); + options.addOption(debugOpt); + options.addOption(pythonOpt); + options.addOptionGroup(fileOrScriptOpt); + options.addOptionGroup(argsOrNVArgsOpt); + options.addOption(helpOpt); + return options; + } /** * Single entry point for all public invocation alternatives (e.g., @@ -231,127 +441,86 @@ public class DMLScript public static boolean executeScript( Configuration conf, String[] args ) throws DMLException { - //Step 1: parse arguments - //check for help - if( args.length==0 || (args.length==1 && (args[0].equalsIgnoreCase("-help")|| args[0].equalsIgnoreCase("-?"))) ){ - System.err.println( USAGE ); - return true; - } - - //check for clean - else if( args.length==1 && args[0].equalsIgnoreCase("-clean") ){ - cleanSystemMLWorkspace(); - return true; - } - - //check number of args - print usage if incorrect - if( args.length < 2 ){ - System.err.println( "ERROR: Unrecognized invocation arguments." ); - System.err.println( USAGE ); - return false; - } - - //check script arg - print usage if incorrect - if (!(args[0].equals("-f") || args[0].equals("-s"))){ - System.err.println("ERROR: First argument must be either -f or -s"); - System.err.println( USAGE ); - return false; - } - //parse arguments and set execution properties - RUNTIME_PLATFORM oldrtplatform = rtplatform; //keep old rtplatform - ExplainType oldexplain = EXPLAIN; //keep old explain - - // Reset global flags to avoid errors in test suite - ENABLE_DEBUG_MODE = false; - - boolean parsePyDML = false; + RUNTIME_PLATFORM oldrtplatform = rtplatform; //keep old rtplatform + ExplainType oldexplain = EXPLAIN; //keep old explain + + Options options = createCLIOptions(); try { - String fnameOptConfig = null; //optional config filename - String[] scriptArgs = null; //optional script arguments - boolean namedScriptArgs = false; - - for( int i=2; i<args.length; i++ ) - { - if( args[i].equalsIgnoreCase("-explain") ) { - EXPLAIN = ExplainType.RUNTIME; - if( args.length > (i+1) && !args[i+1].startsWith("-") ) - EXPLAIN = Explain.parseExplainType(args[++i]); - } - else if( args[i].equalsIgnoreCase("-stats") ) { - STATISTICS = true; - if (args.length > (i + 1) && !args[i + 1].startsWith("-")) - STATISTICS_COUNT = Integer.parseInt(args[++i]); - } - else if ( args[i].equalsIgnoreCase("-exec")) { - rtplatform = parseRuntimePlatform(args[++i]); - if( rtplatform==null ) - return false; - } - else if (args[i].startsWith("-config=")) // legacy - fnameOptConfig = args[i].substring(8).replaceAll("\"", ""); - else if (args[i].equalsIgnoreCase("-config")) - fnameOptConfig = args[++i]; - else if( args[i].equalsIgnoreCase("-debug") ) { - ENABLE_DEBUG_MODE = true; - } - else if( args[i].equalsIgnoreCase("-gpu") ) { - USE_ACCELERATOR = true; - if( args.length > (i+1) && !args[i+1].startsWith("-") ) { - String flag = args[++i]; - if(flag.startsWith("force=")) { - String [] flagOptions = flag.split("="); - if(flagOptions.length == 2) - FORCE_ACCELERATOR = Boolean.parseBoolean(flagOptions[1]); - else - throw new DMLRuntimeException("Unsupported \"force\" option for -gpu:" + flag); - } - else { - throw new DMLRuntimeException("Unsupported flag for -gpu:" + flag); - } - } - GPUContext.getGPUContext(); // creates the singleton GPU context object. Return value ignored. - } - else if( args[i].equalsIgnoreCase("-python") ) { - parsePyDML = true; - } - else if (args[i].startsWith("-args") || args[i].startsWith("-nvargs")) { - namedScriptArgs = args[i].startsWith("-nvargs"); i++; - scriptArgs = new String[args.length - i]; - System.arraycopy(args, i, scriptArgs, 0, scriptArgs.length); - break; - } - else{ - System.err.println("ERROR: Unknown argument: " + args[i]); - return false; - } + DMLOptions dmlOptions = parseCLArguments(args, options); + + // String[] scriptArgs = null; //optional script arguments + // boolean namedScriptArgs = false; + + STATISTICS = dmlOptions.stats; + STATISTICS_COUNT = dmlOptions.statsCount; + USE_ACCELERATOR = dmlOptions.gpu; + FORCE_ACCELERATOR = dmlOptions.forceGPU; + EXPLAIN = dmlOptions.explainType; + ENABLE_DEBUG_MODE = dmlOptions.debug; + SCRIPT_TYPE = dmlOptions.scriptType; + rtplatform = dmlOptions.execMode; + + String fnameOptConfig = dmlOptions.configFile; + boolean isFile = dmlOptions.filePath != null; + String fileOrScript = isFile ? dmlOptions.filePath : dmlOptions.script; + + boolean help = dmlOptions.help; + + if (help) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( "systemml", options ); + return true; } - + + if (USE_ACCELERATOR){ + GPUContext.getGPUContext(); + } + + if (dmlOptions.clean) { + cleanSystemMLWorkspace(); + return true; + } + //set log level if (!ENABLE_DEBUG_MODE) setLoggingProperties( conf ); //Step 2: prepare script invocation - if (StringUtils.endsWithIgnoreCase(args[1], ".pydml")) { - parsePyDML = true; + if (isFile && StringUtils.endsWithIgnoreCase(fileOrScript, ".pydml")) { + SCRIPT_TYPE = ScriptType.PYDML; } - String dmlScriptStr = readDMLScript(args[0], args[1]); - Map<String, String> argVals = createArgumentsMap(namedScriptArgs, scriptArgs); - - DML_FILE_PATH_ANTLR_PARSER = args[1]; + + String dmlScriptStr = readDMLScript(isFile, fileOrScript); + Map<String, String> argVals = dmlOptions.argVals; + + DML_FILE_PATH_ANTLR_PARSER = dmlOptions.filePath; //Step 3: invoke dml script - printInvocationInfo(args[1], fnameOptConfig, argVals); + printInvocationInfo(fileOrScript, fnameOptConfig, argVals); if (ENABLE_DEBUG_MODE) { // inner try loop is just to isolate the debug exception, which will allow to manage the bugs from debugger v/s runtime - launchDebugger(dmlScriptStr, fnameOptConfig, argVals, parsePyDML); + launchDebugger(dmlScriptStr, fnameOptConfig, argVals, SCRIPT_TYPE); } else { - execute(dmlScriptStr, fnameOptConfig, argVals, args, parsePyDML); + execute(dmlScriptStr, fnameOptConfig, argVals, args, SCRIPT_TYPE); } } + catch(AlreadySelectedException e) + { + System.err.println("Mutually exclusive options were selected. " + e.getMessage()); + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( "systemml", options ); + return false; + } + catch(org.apache.commons.cli.ParseException e) + { + System.err.println(e.getMessage()); + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( "systemml", options ); + } catch (ParseException pe) { throw pe; } @@ -378,6 +547,17 @@ public class DMLScript // private internal utils (argument parsing) //////// + @Deprecated() + /** + * Creates an argument map appropriate for consumption by the backend + * The only method using this is the legacy {@link MLContext} api. + * Once that is removed, this function should be removed as well. + * This method uses a fragile position based argument for -args & -nvargs + * @param hasNamedArgs true for named arguments, false for positional arguments + * @param args in "k=v" format for named arguments and "v" for positional arguments + * @return a map containing either ($K,V) or ($1,V) for named and positional arguments respectively + * @throws LanguageException when a named argument is an invalid identifier for DML/PyDML + */ protected static Map<String,String> createArgumentsMap(boolean hasNamedArgs, String[] args) throws LanguageException { @@ -397,7 +577,7 @@ public class DMLScript arg.equalsIgnoreCase("-stats") || arg.equalsIgnoreCase("-exec") || arg.equalsIgnoreCase("-debug") || - arg.startsWith("-config=")) + arg.startsWith("-config")) { throw new LanguageException("-args or -nvargs must be the final argument for DMLScript!"); } @@ -430,17 +610,26 @@ public class DMLScript return argMap; } - - protected static String readDMLScript( String argname, String script ) + + + /** + * Reads the DML/PyDML script into a String + * @param isFile Whether the string argument is a path to a file or the script itself + * @param scriptOrFilename script or filename + * @return a string representation of the script + * @throws IOException if error + * @throws LanguageException if error + */ + protected static String readDMLScript( boolean isFile, String scriptOrFilename ) throws IOException, LanguageException { - boolean fromFile = argname.equals("-f"); String dmlScriptStr; - if( fromFile ) + if( isFile ) { + String fileName = scriptOrFilename; //read DML script from file - if(script == null) + if(fileName == null) throw new LanguageException("DML script path was not specified!"); StringBuilder sb = new StringBuilder(); @@ -448,21 +637,21 @@ public class DMLScript try { //read from hdfs or gpfs file system - if( script.startsWith("hdfs:") - || script.startsWith("gpfs:") ) + if( fileName.startsWith("hdfs:") + || fileName.startsWith("gpfs:") ) { - if( !LocalFileUtils.validateExternalFilename(script, true) ) + if( !LocalFileUtils.validateExternalFilename(fileName, true) ) throw new LanguageException("Invalid (non-trustworthy) hdfs filename."); FileSystem fs = FileSystem.get(ConfigurationManager.getCachedJobConf()); - Path scriptPath = new Path(script); + Path scriptPath = new Path(fileName); in = new BufferedReader(new InputStreamReader(fs.open(scriptPath))); } // from local file system else { - if( !LocalFileUtils.validateExternalFilename(script, false) ) + if( !LocalFileUtils.validateExternalFilename(fileName, false) ) throw new LanguageException("Invalid (non-trustworthy) local filename."); - in = new BufferedReader(new FileReader(script)); + in = new BufferedReader(new FileReader(fileName)); } //core script reading @@ -486,11 +675,12 @@ public class DMLScript } else { + String scriptString = scriptOrFilename; //parse given script string - if(script == null) + if(scriptString == null) throw new LanguageException("DML script was not specified!"); - InputStream is = new ByteArrayInputStream(script.getBytes()); + InputStream is = new ByteArrayInputStream(scriptString.getBytes()); Scanner scan = new Scanner(is); dmlScriptStr = scan.useDelimiter("\\A").next(); scan.close(); @@ -498,26 +688,7 @@ public class DMLScript return dmlScriptStr; } - - private static RUNTIME_PLATFORM parseRuntimePlatform( String platform ) - { - RUNTIME_PLATFORM lrtplatform = null; - - if ( platform.equalsIgnoreCase("hadoop")) - lrtplatform = RUNTIME_PLATFORM.HADOOP; - else if ( platform.equalsIgnoreCase("singlenode")) - lrtplatform = RUNTIME_PLATFORM.SINGLE_NODE; - else if ( platform.equalsIgnoreCase("hybrid")) - lrtplatform = RUNTIME_PLATFORM.HYBRID; - else if ( platform.equalsIgnoreCase("spark")) - lrtplatform = RUNTIME_PLATFORM.SPARK; - else if ( platform.equalsIgnoreCase("hybrid_spark")) - lrtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; - else - System.err.println("ERROR: Unknown runtime platform: " + platform); - - return lrtplatform; - } + private static void setLoggingProperties( Configuration conf ) { @@ -550,7 +721,7 @@ public class DMLScript * @param fnameOptConfig configuration file * @param argVals map of argument values * @param allArgs arguments - * @param parsePyDML true if PYDML, false if DML + * @param scriptType type of script (DML or PyDML) * @throws ParseException if ParseException occurs * @throws IOException if IOException occurs * @throws DMLRuntimeException if DMLRuntimeException occurs @@ -558,10 +729,10 @@ public class DMLScript * @throws HopsException if HopsException occurs * @throws LopsException if LopsException occurs */ - private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs, boolean parsePyDML) + private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs, ScriptType scriptType) throws ParseException, IOException, DMLRuntimeException, LanguageException, HopsException, LopsException { - SCRIPT_TYPE = parsePyDML ? ScriptType.PYDML : ScriptType.DML; + SCRIPT_TYPE = scriptType; //print basic time and environment info printStartExecInfo( dmlScriptStr ); @@ -580,7 +751,7 @@ public class DMLScript //Step 3: parse dml script Statistics.startCompileTimer(); - AParserWrapper parser = AParserWrapper.createParser(parsePyDML); + AParserWrapper parser = AParserWrapper.createParser(scriptType); DMLProgram prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals); //Step 4: construct HOP DAGs (incl LVA, validate, and setup) @@ -690,7 +861,7 @@ public class DMLScript * @param dmlScriptStr DML script contents (including new lines) * @param fnameOptConfig Full path of configuration file for SystemML * @param argVals Key-value pairs defining arguments of DML script - * @param parsePyDML true if PYDML, false if DML + * @param scriptType type of script (DML or PyDML) * @throws ParseException if ParseException occurs * @throws IOException if IOException occurs * @throws DMLRuntimeException if DMLRuntimeException occurs @@ -699,7 +870,7 @@ public class DMLScript * @throws HopsException if HopsException occurs * @throws LopsException if LopsException occurs */ - private static void launchDebugger(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, boolean parsePyDML) + private static void launchDebugger(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, ScriptType scriptType) throws ParseException, IOException, DMLRuntimeException, DMLDebuggerException, LanguageException, HopsException, LopsException { DMLDebuggerProgramInfo dbprog = new DMLDebuggerProgramInfo(); @@ -709,7 +880,7 @@ public class DMLScript ConfigurationManager.setGlobalConfig(conf); //Step 2: parse dml script - AParserWrapper parser = AParserWrapper.createParser(parsePyDML); + AParserWrapper parser = AParserWrapper.createParser(scriptType); DMLProgram prog = parser.parse(DML_FILE_PATH_ANTLR_PARSER, dmlScriptStr, argVals); //Step 3: construct HOP DAGs (incl LVA and validate)
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/api/MLContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/MLContext.java b/src/main/java/org/apache/sysml/api/MLContext.java index 6382832..0bfab39 100644 --- a/src/main/java/org/apache/sysml/api/MLContext.java +++ b/src/main/java/org/apache/sysml/api/MLContext.java @@ -20,14 +20,6 @@ package org.apache.sysml.api; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Scanner; - import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.spark.SparkContext; @@ -41,7 +33,6 @@ import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.SparkSession; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.api.jmlc.JMLCUtils; -import org.apache.sysml.api.mlcontext.ScriptType; import org.apache.sysml.conf.CompilerConfig; import org.apache.sysml.conf.CompilerConfig.ConfigType; import org.apache.sysml.conf.ConfigurationManager; @@ -92,6 +83,14 @@ import org.apache.sysml.utils.Explain; import org.apache.sysml.utils.Explain.ExplainCounts; import org.apache.sysml.utils.Statistics; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Scanner; + /** * MLContext is useful for passing RDDs as input/output to SystemML. This API avoids the need to read/write * from HDFS (which is another way to pass inputs to SystemML). @@ -770,7 +769,7 @@ public class MLContext { args[i] = entry.getKey() + "=" + entry.getValue(); i++; } - return compileAndExecuteScript(dmlScriptFilePath, args, true, parsePyDML, configFilePath); + return compileAndExecuteScript(dmlScriptFilePath, args, true, parsePyDML ? ScriptType.PYDML : ScriptType.DML, configFilePath); } /** @@ -784,17 +783,7 @@ public class MLContext { * @throws ParseException if ParseException occurs */ public MLOutput execute(String dmlScriptFilePath, Map<String, String> namedArgs, String configFilePath) throws IOException, DMLException, ParseException { - String [] args = new String[namedArgs.size()]; - int i = 0; - for(Entry<String, String> entry : namedArgs.entrySet()) { - if(entry.getValue().trim().isEmpty()) - args[i] = entry.getKey() + "=\"" + entry.getValue() + "\""; - else - args[i] = entry.getKey() + "=" + entry.getValue(); - i++; - } - - return compileAndExecuteScript(dmlScriptFilePath, args, true, false, configFilePath); + return execute(dmlScriptFilePath, namedArgs, false, configFilePath); } /** @@ -1013,7 +1002,7 @@ public class MLContext { * @throws ParseException if ParseException occurs */ public MLOutput execute(String dmlScriptFilePath, String [] args, boolean parsePyDML, String configFilePath) throws IOException, DMLException, ParseException { - return compileAndExecuteScript(dmlScriptFilePath, args, false, parsePyDML, configFilePath); + return compileAndExecuteScript(dmlScriptFilePath, args, false, parsePyDML ? ScriptType.PYDML : ScriptType.DML, configFilePath); } /** @@ -1066,7 +1055,7 @@ public class MLContext { * @throws ParseException if ParseException occurs */ public MLOutput execute(String dmlScriptFilePath, boolean parsePyDML, String configFilePath) throws IOException, DMLException, ParseException { - return compileAndExecuteScript(dmlScriptFilePath, null, false, parsePyDML, configFilePath); + return compileAndExecuteScript(dmlScriptFilePath, null, false, parsePyDML ? ScriptType.PYDML : ScriptType.DML, configFilePath); } /** @@ -1313,7 +1302,7 @@ public class MLContext { public MLOutput executeScript(String dmlScript, boolean isPyDML, String configFilePath) throws IOException, DMLException { - return compileAndExecuteScript(dmlScript, null, false, false, isPyDML, configFilePath); + return compileAndExecuteScript(dmlScript, null, false, false, isPyDML ? ScriptType.PYDML : ScriptType.DML, configFilePath); } /* @@ -1390,7 +1379,7 @@ public class MLContext { args[i] = entry.getKey() + "=" + entry.getValue(); i++; } - return compileAndExecuteScript(dmlScript, args, false, true, isPyDML, configFilePath); + return compileAndExecuteScript(dmlScript, args, false, true, isPyDML ? ScriptType.PYDML : ScriptType.DML, configFilePath); } private void checkIfRegisteringInputAllowed() throws DMLRuntimeException { @@ -1399,26 +1388,29 @@ public class MLContext { } } - private MLOutput compileAndExecuteScript(String dmlScriptFilePath, String [] args, boolean isNamedArgument, boolean isPyDML, String configFilePath) throws IOException, DMLException { - return compileAndExecuteScript(dmlScriptFilePath, args, true, isNamedArgument, isPyDML, configFilePath); + private MLOutput compileAndExecuteScript(String dmlScriptFilePath, String [] args, boolean isNamedArgument, ScriptType scriptType, String configFilePath) throws IOException, DMLException { + return compileAndExecuteScript(dmlScriptFilePath, args, true, isNamedArgument, scriptType, configFilePath); } - + /** * All the execute() methods call this, which after setting appropriate input/output variables * calls _compileAndExecuteScript * We have explicitly synchronized this function because MLContext/SystemML does not yet support multi-threading. + * @throws ParseException if ParseException occurs * @param dmlScriptFilePath script file path * @param args arguments - * @param isNamedArgument is named argument + * @param isFile whether the string is a path + * @param isNamedArgument is named argument + * @param scriptType type of script (DML or PyDML) + * @param configFilePath path to config file * @return output as MLOutput * @throws IOException if IOException occurs * @throws DMLException if DMLException occurs - * @throws ParseException if ParseException occurs */ - private synchronized MLOutput compileAndExecuteScript(String dmlScriptFilePath, String [] args, boolean isFile, boolean isNamedArgument, boolean isPyDML, String configFilePath) throws IOException, DMLException { + private synchronized MLOutput compileAndExecuteScript(String dmlScriptFilePath, String [] args, boolean isFile, boolean isNamedArgument, ScriptType scriptType, String configFilePath) throws IOException, DMLException { try { - DMLScript.SCRIPT_TYPE = isPyDML ? ScriptType.PYDML : ScriptType.DML; + DMLScript.SCRIPT_TYPE = scriptType; if(getActiveMLContext() != null) { throw new DMLRuntimeException("SystemML (and hence by definition MLContext) doesnot support parallel execute() calls from same or different MLContexts. " @@ -1438,7 +1430,7 @@ public class MLContext { Map<String, String> argVals = DMLScript.createArgumentsMap(isNamedArgument, args); // Run the DML script - ExecutionContext ec = executeUsingSimplifiedCompilationChain(dmlScriptFilePath, isFile, argVals, isPyDML, inputs, outputs, _variables, configFilePath); + ExecutionContext ec = executeUsingSimplifiedCompilationChain(dmlScriptFilePath, isFile, argVals, scriptType, inputs, outputs, _variables, configFilePath); SparkExecutionContext sec = (SparkExecutionContext) ec; // Now collect the output @@ -1481,7 +1473,7 @@ public class MLContext { * @param dmlScriptFilePath script file path * @param isFile true if file, false otherwise * @param argVals map of args - * @param parsePyDML true if pydml, false otherwise + * @param scriptType type of script (DML or PyDML) * @param inputs the inputs * @param outputs the outputs * @param inputSymbolTable the input symbol table @@ -1491,7 +1483,7 @@ public class MLContext { * @throws DMLException if DMLException occurs * @throws ParseException if ParseException occurs */ - private ExecutionContext executeUsingSimplifiedCompilationChain(String dmlScriptFilePath, boolean isFile, Map<String, String> argVals, boolean parsePyDML, + private ExecutionContext executeUsingSimplifiedCompilationChain(String dmlScriptFilePath, boolean isFile, Map<String, String> argVals, ScriptType scriptType, String[] inputs, String[] outputs, LocalVariableMap inputSymbolTable, String configFilePath) throws IOException, DMLException { @@ -1510,13 +1502,13 @@ public class MLContext { ConfigurationManager.setGlobalConfig(cconf); //read dml script string - String dmlScriptStr = DMLScript.readDMLScript( isFile?"-f":"-s", dmlScriptFilePath); + String dmlScriptStr = DMLScript.readDMLScript( isFile, dmlScriptFilePath); //simplified compilation chain _rtprog = null; //parsing - AParserWrapper parser = AParserWrapper.createParser(parsePyDML); + AParserWrapper parser = AParserWrapper.createParser(scriptType); DMLProgram prog; if (isFile) { prog = parser.parse(dmlScriptFilePath, null, argVals); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/api/ScriptType.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/ScriptType.java b/src/main/java/org/apache/sysml/api/ScriptType.java new file mode 100644 index 0000000..00ac95e --- /dev/null +++ b/src/main/java/org/apache/sysml/api/ScriptType.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.api; + +/** + * ScriptType represents the type of script, DML (R-like syntax) or PYDML + * (Python-like syntax). + * + */ +public enum ScriptType { + /** + * R-like syntax. + */ + DML, + + /** + * Python-like syntax. + */ + PYDML; + + /** + * Obtain script type as a lowercase string ("dml" or "pydml"). + * + * @return lowercase string representing the script type + */ + public String lowerCase() { + return super.toString().toLowerCase(); + } + + /** + * Is the script type DML? + * + * @return {@code true} if the script type is DML, {@code false} otherwise + */ + public boolean isDML() { + return (this == ScriptType.DML); + } + + /** + * Is the script type PYDML? + * + * @return {@code true} if the script type is PYDML, {@code false} otherwise + */ + public boolean isPYDML() { + return (this == ScriptType.PYDML); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/api/jmlc/Connection.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java index a3d7ae7..c5bd265 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java +++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java @@ -19,21 +19,12 @@ package org.apache.sysml.api.jmlc; -import java.io.BufferedReader; -import java.io.Closeable; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.Map; - import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.sysml.api.DMLException; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.api.mlcontext.ScriptType; +import org.apache.sysml.api.ScriptType; import org.apache.sysml.conf.CompilerConfig; import org.apache.sysml.conf.CompilerConfig.ConfigType; import org.apache.sysml.conf.ConfigurationManager; @@ -63,6 +54,15 @@ import org.apache.sysml.runtime.transform.meta.TfMetaUtils; import org.apache.sysml.runtime.util.DataConverter; import org.apache.wink.json4j.JSONObject; +import java.io.BufferedReader; +import java.io.Closeable; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.HashMap; +import java.util.Map; + /** * Interaction with SystemML using the JMLC (Java Machine Learning Connector) API is initiated with * a {@link Connection} object. The JMLC API is patterned @@ -167,7 +167,7 @@ public class Connection implements Closeable try { //parsing - AParserWrapper parser = AParserWrapper.createParser(parsePyDML); + AParserWrapper parser = AParserWrapper.createParser(parsePyDML ? ScriptType.PYDML : ScriptType.DML); DMLProgram prog = parser.parse(null, script, args); //language validate http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java index 17aa4a7..dada3d1 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContextUtil.java @@ -51,6 +51,7 @@ import org.apache.spark.sql.types.DataType; import org.apache.spark.sql.types.DataTypes; import org.apache.spark.sql.types.StructField; import org.apache.spark.sql.types.StructType; +import org.apache.sysml.api.ScriptType; import org.apache.sysml.conf.CompilerConfig; import org.apache.sysml.conf.CompilerConfig.ConfigType; import org.apache.sysml.conf.ConfigurationManager; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/api/mlcontext/Script.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/Script.java b/src/main/java/org/apache/sysml/api/mlcontext/Script.java index 54903b7..0e5e4eb 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/Script.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/Script.java @@ -28,6 +28,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import org.apache.sysml.api.ScriptType; import org.apache.sysml.runtime.controlprogram.LocalVariableMap; import org.apache.sysml.runtime.controlprogram.caching.CacheableData; import org.apache.sysml.runtime.instructions.cp.Data; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java index c5801d4..c49ca85 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java @@ -19,10 +19,6 @@ package org.apache.sysml.api.mlcontext; -import java.io.IOException; -import java.util.Map; -import java.util.Set; - import org.apache.commons.lang3.StringUtils; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.jmlc.JMLCUtils; @@ -52,6 +48,10 @@ import org.apache.sysml.utils.Explain.ExplainCounts; import org.apache.sysml.utils.Explain.ExplainType; import org.apache.sysml.utils.Statistics; +import java.io.IOException; +import java.util.Map; +import java.util.Set; + /** * ScriptExecutor executes a DML or PYDML Script object using SystemML. This is * accomplished by calling the {@link #execute} method. @@ -431,7 +431,7 @@ public class ScriptExecutor { */ protected void parseScript() { try { - AParserWrapper parser = AParserWrapper.createParser(script.getScriptType().isPYDML()); + AParserWrapper parser = AParserWrapper.createParser(script.getScriptType()); Map<String, Object> inputParameters = script.getInputParameters(); Map<String, String> inputParametersStringMaps = MLContextUtil.convertInputParametersForParser( inputParameters, script.getScriptType()); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java index db89bd1..87579c9 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java @@ -30,6 +30,7 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.sysml.api.ScriptType; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.runtime.util.LocalFileUtils; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/api/mlcontext/ScriptType.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptType.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptType.java deleted file mode 100644 index 94c9057..0000000 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptType.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.api.mlcontext; - -/** - * ScriptType represents the type of script, DML (R-like syntax) or PYDML - * (Python-like syntax). - * - */ -public enum ScriptType { - /** - * R-like syntax. - */ - DML, - - /** - * Python-like syntax. - */ - PYDML; - - /** - * Obtain script type as a lowercase string ("dml" or "pydml"). - * - * @return lowercase string representing the script type - */ - public String lowerCase() { - return super.toString().toLowerCase(); - } - - /** - * Is the script type DML? - * - * @return {@code true} if the script type is DML, {@code false} otherwise - */ - public boolean isDML() { - return (this == ScriptType.DML); - } - - /** - * Is the script type PYDML? - * - * @return {@code true} if the script type is PYDML, {@code false} otherwise - */ - public boolean isPYDML() { - return (this == ScriptType.PYDML); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/parser/AParserWrapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/AParserWrapper.java b/src/main/java/org/apache/sysml/parser/AParserWrapper.java index a413f53..e3e4cdb 100644 --- a/src/main/java/org/apache/sysml/parser/AParserWrapper.java +++ b/src/main/java/org/apache/sysml/parser/AParserWrapper.java @@ -19,16 +19,10 @@ package org.apache.sysml.parser; -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.List; -import java.util.Map; - import org.apache.commons.logging.Log; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.sysml.api.ScriptType; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.parser.common.CommonSyntacticValidator; import org.apache.sysml.parser.common.CustomErrorListener.ParseIssue; @@ -37,6 +31,13 @@ import org.apache.sysml.parser.pydml.PyDMLParserWrapper; import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.util.LocalFileUtils; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.List; +import java.util.Map; + /** * Base class for all dml parsers in order to make the various compilation chains * independent of the used parser. @@ -54,19 +55,18 @@ public abstract class AParserWrapper /** * Factory method for creating parser wrappers * - * @param pydml true if a PyDML parser is needed + * @param scriptType type of script * @return parser wrapper */ - public static AParserWrapper createParser(boolean pydml) + public static AParserWrapper createParser(ScriptType scriptType) { AParserWrapper ret = null; //create the parser instance - if( pydml ) - ret = new PyDMLParserWrapper(); - else - ret = new DMLParserWrapper(); - + switch (scriptType) { + case DML: ret = new DMLParserWrapper(); break; + case PYDML: ret = new PyDMLParserWrapper(); break; + } CommonSyntacticValidator.init(); return ret; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java index dff1339..06df216 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BooleanObject.java @@ -21,7 +21,7 @@ package org.apache.sysml.runtime.instructions.cp; import org.apache.commons.lang.StringUtils; import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.mlcontext.ScriptType; +import org.apache.sysml.api.ScriptType; import org.apache.sysml.parser.Expression.ValueType; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java b/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java index bd9116d..32471a3 100644 --- a/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java +++ b/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java @@ -19,14 +19,6 @@ package org.apache.sysml.yarn; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FSDataOutputStream; @@ -52,7 +44,6 @@ import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; - import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.parser.ParseException; import org.apache.sysml.runtime.DMLRuntimeException; @@ -60,6 +51,14 @@ import org.apache.sysml.runtime.DMLScriptException; import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing; import org.apache.sysml.runtime.util.MapReduceTool; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + /** * NOTES: * * Security: By default, submitted applications are ran as user 'yarn'. @@ -429,7 +428,7 @@ public class DMLYarnClient command.append(' '); if( i>0 && _args[i-1].equals("-f") ){ command.append(_hdfsDMLScript); - command.append(" -config=" + _hdfsDMLConfig); + command.append(" -config " + _hdfsDMLConfig); } else if( _args[i].startsWith("-config") ){ //ignore because config added with -f http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/resources/scripts/sparkDML.sh ---------------------------------------------------------------------- diff --git a/src/main/resources/scripts/sparkDML.sh b/src/main/resources/scripts/sparkDML.sh index cd57ae0..a68d34a 100644 --- a/src/main/resources/scripts/sparkDML.sh +++ b/src/main/resources/scripts/sparkDML.sh @@ -116,7 +116,7 @@ $SPARK_HOME/bin/spark-submit \ ${conf} \ ${SYSTEMML_HOME}/${project.artifactId}-${project.version}.jar \ -f ${f} \ - -config=${SYSTEMML_HOME}/SystemML-config.xml \ + -config ${SYSTEMML_HOME}/SystemML-config.xml \ -exec hybrid_spark \ $explain \ $stats \ http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/standalone/runStandaloneSystemML.bat ---------------------------------------------------------------------- diff --git a/src/main/standalone/runStandaloneSystemML.bat b/src/main/standalone/runStandaloneSystemML.bat index 9275d9e..35ebbc7 100644 --- a/src/main/standalone/runStandaloneSystemML.bat +++ b/src/main/standalone/runStandaloneSystemML.bat @@ -46,7 +46,7 @@ set CMD=java %SYSTEMML_STANDALONE_OPTS% ^ org.apache.sysml.api.DMLScript ^ -f %1 ^ -exec singlenode ^ - -config=SystemML-config.xml ^ + -config SystemML-config.xml ^ %ALLBUTFIRST% :: execute the java command @@ -72,6 +72,6 @@ GOTO Msg :Msg ECHO Usage: runStandaloneSystemML.bat ^<dml-filename^> [arguments] [-help] ECHO Default Java options (-Xmx4g -Xms4g -Xmn400m) can be overridden by setting SYSTEMML_STANDALONE_OPTS. -ECHO Script internally invokes 'java [SYSTEMML_STANDALONE_OPTS] -cp ./lib/* -Dlog4j.configuration=file:log4j.properties org.apache.sysml.api.DMLScript -f ^<dml-filename^> -exec singlenode -config=SystemML-config.xml [arguments]' +ECHO Script internally invokes 'java [SYSTEMML_STANDALONE_OPTS] -cp ./lib/* -Dlog4j.configuration=file:log4j.properties org.apache.sysml.api.DMLScript -f ^<dml-filename^> -exec singlenode -config SystemML-config.xml [arguments]' :End http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/main/standalone/runStandaloneSystemML.sh ---------------------------------------------------------------------- diff --git a/src/main/standalone/runStandaloneSystemML.sh b/src/main/standalone/runStandaloneSystemML.sh index 28000b4..2980080 100644 --- a/src/main/standalone/runStandaloneSystemML.sh +++ b/src/main/standalone/runStandaloneSystemML.sh @@ -76,7 +76,7 @@ java ${SYSTEMML_STANDALONE_OPTS} \ org.apache.sysml.api.DMLScript \ -f ${SCRIPT_FILE} \ -exec singlenode \ --config=$CURRENT_PATH"/SystemML-config.xml" \ +-config $CURRENT_PATH"/SystemML-config.xml" \ $@" $CMD http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java index ffb080b..cd85efe 100644 --- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java @@ -1217,7 +1217,8 @@ public abstract class AutomatedTestBase throw new RuntimeException("Unknown runtime platform: " + rtplatform); } //use optional config file since default under SystemML/DML - args.add("-config="+ getCurConfigFile().getPath()); + args.add("-config"); + args.add(getCurConfigFile().getPath()); if(TEST_GPU) args.add("-gpu"); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest1.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest1.java b/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest1.java deleted file mode 100644 index c7006df..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest1.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.dmlscript; - -import org.junit.Test; - -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; - - -/** - * <p> - * <b>Positive tests:</b> - * </p> - * <ul> - * <li>text format</li> - * <li>binary format</li> - * </ul> - * <p> - * <b>Negative tests:</b> - * </p> - * <ul> - * </ul> - * - * - */ -public class DMLScriptTest1 extends AutomatedTestBase -{ - - private final static String TEST_DIR = "functions/dmlscript/"; - private final static String TEST_CLASS_DIR = TEST_DIR + DMLScriptTest1.class.getSimpleName() + "/"; - private final static String TEST_NAME = "DMLScriptTest"; - - @Override - public void setUp() { - // positive tests - TestConfiguration config = new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] { "a" }); - addTestConfiguration(TEST_NAME, config); - - // negative tests - } - - @Test - public void testWithFile() { - int rows = 10; - int cols = 10; - - TestConfiguration config = getTestConfiguration(TEST_NAME); - config.addVariable("rows", rows); - config.addVariable("cols", cols); - config.addVariable("format", "text"); - loadTestConfiguration(config); - - String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + "DMLScriptTest.dml"; - - programArgs = new String[]{"-args", input("a"), - Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - - double[][] a = getRandomMatrix(rows, cols, -1, 1, 0.5, -1); - writeInputMatrix("a", a, true); - - runTest(true, false, null, -1); - - programArgs = new String[]{"-args", input("a"), - Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - - programArgs = new String[]{"-exec", "hybrid", "-args", input("a"), - Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - - programArgs = new String[]{"-exec", "hybrid", "-config=" + HOME + "SystemML-config.xml", - "-args", input("a"), Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - } - - @Test - public void testWithString() { - String s = " A = read($1, rows=$2, cols=$3, format=$4); \n " + - "write(A, $5, format=$4); \n"; - int rows = 10; - int cols = 10; - String HOME = SCRIPT_DIR + TEST_DIR; - - TestConfiguration config = getTestConfiguration(TEST_NAME); - config.addVariable("rows", rows); - config.addVariable("cols", cols); - config.addVariable("format", "text"); - loadTestConfiguration(config); - - programArgs = new String[]{"-s", s, "-args", input("a"), - Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - - double[][] a = getRandomMatrix(rows, cols, -1, 1, 0.5, -1); - writeInputMatrix("a", a, true); - - runTest(true, false, null, -1); - - programArgs = new String[]{"-s", s, "-args", input("a"), - Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - - programArgs = new String[]{"-s", s, "-config=" + HOME + "SystemML-config.xml", "-exec", "hybrid", - "-args", input("a"), Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - } -} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest2.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest2.java b/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest2.java deleted file mode 100644 index 123146c..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/dmlscript/DMLScriptTest2.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.dmlscript; - -import org.junit.Test; - -import org.apache.sysml.api.DMLException; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; - - -/** - * <p> - * <b>Positive tests:</b> - * </p> - * <ul> - * <li>text format</li> - * <li>binary format</li> - * </ul> - * <p> - * <b>Negative tests:</b> - * </p> - * <ul> - * </ul> - * - * - */ -public class DMLScriptTest2 extends AutomatedTestBase -{ - - private final static String TEST_DIR = "functions/dmlscript/"; - private final static String TEST_CLASS_DIR = TEST_DIR + DMLScriptTest2.class.getSimpleName() + "/"; - private final static String TEST_NAME = "DMLScriptTest2"; - - /** - * Main method for running one test at a time. - */ - public static void main(String[] args) { - long startMsec = System.currentTimeMillis(); - - DMLScriptTest2 t = new DMLScriptTest2(); - t.setUpBase(); - t.setUp(); - t.testWithString(); - t.tearDown(); - - long elapsedMsec = System.currentTimeMillis() - startMsec; - System.err.printf("Finished in %1.3f sec\n", elapsedMsec / 1000.0); - } - - @Override - public void setUp() { - // positive tests - - // negative tests - TestConfiguration config = new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] { "a" }); - addTestConfiguration(TEST_NAME, config); - } - - @Test - public void testWithFile() { - int rows = 10; - int cols = 10; - String HOME = SCRIPT_DIR + TEST_DIR; - - TestConfiguration config = getTestConfiguration(TEST_NAME); - config.addVariable("rows", rows); - config.addVariable("cols", cols); - config.addVariable("format", "text"); - loadTestConfiguration(config); - - double[][] a = getRandomMatrix(rows, cols, -1, 1, 0.5, -1); - writeInputMatrix("a", a, true); - - //Expect to print out an ERROR message. -f or -s must be the first argument. - fullDMLScriptName = HOME + "DMLScriptTest.dml"; - programArgs = new String[]{ "-exec", "hybrid", "-args", input("a"), - Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - - //Expect to print out an ERROR message. -args should be the last argument. - programArgs = new String[]{"-args", input("a"), - Integer.toString(rows), Integer.toString(cols), "text", output("a"), "-exec", "hybrid"}; - runTest(true, true, DMLException.class, -1); - - //Expect to print out an ERROR message, -de is an unknown argument - programArgs = new String[]{"-de", "-exec", "hybrid", "-config=" + HOME + "SystemML-config.xml", - "-args", input("a"), Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - - //Expect to print out an ERROR message, -config syntax is -config=<config file> - programArgs = new String[]{"-exec", "hybrid", "-config", HOME + "SystemML-config.xml", - "-args", input("a"), Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - } - - @Test - public void testWithString() { - String s = " A = read($1, rows=$2, cols=$3, format=$4); \n " + - "write(A, $5, format=$4); \n"; - int rows = 10; - int cols = 10; - String HOME = SCRIPT_DIR + TEST_DIR; - - TestConfiguration config = availableTestConfigurations.get("DMLScriptTest2"); - config.addVariable("rows", rows); - config.addVariable("cols", cols); - config.addVariable("format", "text"); - loadTestConfiguration(config); - - double[][] a = getRandomMatrix(rows, cols, -1, 1, 0.5, -1); - writeInputMatrix("a", a, true); - - //Expect to print out an ERROR message. -f or -s must be the first argument. - programArgs = new String[]{ "-v", "-s", s, "-args", input("a"), - Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - - //Expect to print out an ERROR message. -args should be the last argument. - programArgs = new String[]{"-s", s, "-args", "-v", input("a"), - Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - - //Expect to print out an ERROR message, -de is an unknown argument - programArgs = new String[]{"-s", s, "-de", "-args", input("a"), - Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - - //Expect to print out an ERROR message, -config syntax is -config=<config file> - programArgs = new String[]{"-s", s, "-config", HOME + "SystemML-config.xml", "-exec", "hybrid", - "-args", input("a"), Integer.toString(rows), Integer.toString(cols), "text", output("a")}; - runTest(true, false, null, -1); - } -} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java index 42e6c05..76dbfda 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/DataTypeChangeTest.java @@ -19,13 +19,6 @@ package org.apache.sysml.test.integration.functions.misc; -import java.io.BufferedReader; -import java.io.FileReader; -import java.util.HashMap; - -import org.junit.Assert; -import org.junit.Test; - import org.apache.sysml.api.DMLException; import org.apache.sysml.api.DMLScript; import org.apache.sysml.conf.ConfigurationManager; @@ -36,6 +29,12 @@ import org.apache.sysml.parser.DMLTranslator; import org.apache.sysml.parser.LanguageException; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.test.integration.TestConfiguration; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.util.HashMap; /** * GENERAL NOTE @@ -148,11 +147,7 @@ public class DataTypeChangeTest extends AutomatedTestBase public void testDataTypeChangeValidate4f() { runTest("dt_change_4f", false); } - /** - * - * @param cfc - * @param vt - */ + private void runTest( String testName, boolean exceptionExpected ) { String RI_HOME = SCRIPT_DIR + TEST_DIR; @@ -166,11 +161,7 @@ public class DataTypeChangeTest extends AutomatedTestBase runTest(true, exceptionExpected, DMLException.class, -1); } - /** - * - * @param scriptFilename - * @param expectedException - */ + private void runValidateTest( String fullTestName, boolean expectedException ) { boolean raisedException = false; @@ -197,7 +188,7 @@ public class DataTypeChangeTest extends AutomatedTestBase } //parsing and dependency analysis - AParserWrapper parser = AParserWrapper.createParser(false); + AParserWrapper parser = AParserWrapper.createParser(org.apache.sysml.api.ScriptType.DML); DMLProgram prog = parser.parse(DMLScript.DML_FILE_PATH_ANTLR_PARSER, dmlScriptString, argVals); DMLTranslator dmlt = new DMLTranslator(prog); dmlt.liveVariableAnalysis(prog); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java index f58d747..1721a89 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java @@ -19,13 +19,6 @@ package org.apache.sysml.test.integration.functions.parfor; -import java.io.BufferedReader; -import java.io.FileReader; -import java.util.HashMap; - -import org.junit.Assert; -import org.junit.Test; - import org.apache.sysml.api.DMLScript; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; @@ -35,6 +28,12 @@ import org.apache.sysml.parser.DMLTranslator; import org.apache.sysml.parser.LanguageException; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.test.integration.TestConfiguration; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.util.HashMap; /** * Different test cases for ParFOR loop dependency analysis: @@ -351,7 +350,7 @@ public class ParForDependencyAnalysisTest extends AutomatedTestBase } //parsing and dependency analysis - AParserWrapper parser = AParserWrapper.createParser(false); + AParserWrapper parser = AParserWrapper.createParser(org.apache.sysml.api.ScriptType.DML); DMLProgram prog = parser.parse(DMLScript.DML_FILE_PATH_ANTLR_PARSER, dmlScriptString, argVals); DMLTranslator dmlt = new DMLTranslator(prog); dmlt.validateParseTree(prog); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java index 68b434b..0bc5adb 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/transform/TransformFrameEncodeDecodeTest.java @@ -161,8 +161,15 @@ public class TransformFrameEncodeDecodeTest extends AutomatedTestBase "DATA=" + HOME + "input/" + DATASET, "TFSPEC=" + HOME + "input/" + SPEC, "TFDATA=" + output("tfout"), "SEP=,", - "OFMT=" + ofmt, "OSEP=\",\"" }; - + "OFMT=" + ofmt, "OSEP=," }; + + // Originally OSEP was set to + // OSEP="," + // Apache Commons CLI strips away the leading and trailing quotes, leaving us with + // OSEP=", + // This is just a feature/bug and is reported in CLI-262, + // though even a fix is unlikely to be backported to 1.2 + OptimizerUtils.ALLOW_FRAME_CSV_REBLOCK = true; runTest(true, false, null, -1); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/346d1c01/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java index c8d3450..cf91a36 100644 --- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java +++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java @@ -225,7 +225,7 @@ public class MLContextTest extends AutomatedTestBase { System.out.println("MLContextTest - execute DML script"); String testString = "hello dml world!"; setExpectedStdOut(testString); - Script script = new Script("print('" + testString + "');", org.apache.sysml.api.mlcontext.ScriptType.DML); + Script script = new Script("print('" + testString + "');", org.apache.sysml.api.ScriptType.DML); ml.execute(script); } @@ -234,7 +234,7 @@ public class MLContextTest extends AutomatedTestBase { System.out.println("MLContextTest - execute PYDML script"); String testString = "hello pydml world!"; setExpectedStdOut(testString); - Script script = new Script("print('" + testString + "')", org.apache.sysml.api.mlcontext.ScriptType.PYDML); + Script script = new Script("print('" + testString + "')", org.apache.sysml.api.ScriptType.PYDML); ml.execute(script); } @@ -403,7 +403,7 @@ public class MLContextTest extends AutomatedTestBase { System.out.println("MLContextTest - custom execution step DML"); String testString = "custom execution step"; setExpectedStdOut(testString); - Script script = new Script("print('" + testString + "');", org.apache.sysml.api.mlcontext.ScriptType.DML); + Script script = new Script("print('" + testString + "');", org.apache.sysml.api.ScriptType.DML); ScriptExecutor scriptExecutor = new ScriptExecutor() { // turn off global data flow optimization check @@ -420,7 +420,7 @@ public class MLContextTest extends AutomatedTestBase { System.out.println("MLContextTest - custom execution step PYDML"); String testString = "custom execution step"; setExpectedStdOut(testString); - Script script = new Script("print('" + testString + "')", org.apache.sysml.api.mlcontext.ScriptType.PYDML); + Script script = new Script("print('" + testString + "')", org.apache.sysml.api.ScriptType.PYDML); ScriptExecutor scriptExecutor = new ScriptExecutor() { // turn off global data flow optimization check @@ -1558,7 +1558,7 @@ public class MLContextTest extends AutomatedTestBase { public void testOneScriptTwoExecutionsDML() { System.out.println("MLContextTest - one script with two executions DML"); - Script script = new Script(org.apache.sysml.api.mlcontext.ScriptType.DML); + Script script = new Script(org.apache.sysml.api.ScriptType.DML); double[][] m1 = new double[][] { { 1.0, 2.0 }, { 3.0, 4.0 } }; script.setScriptString("sum1 = sum(m1);").in("m1", m1).out("sum1"); @@ -1577,7 +1577,7 @@ public class MLContextTest extends AutomatedTestBase { public void testOneScriptTwoExecutionsPYDML() { System.out.println("MLContextTest - one script with two executions PYDML"); - Script script = new Script(org.apache.sysml.api.mlcontext.ScriptType.PYDML); + Script script = new Script(org.apache.sysml.api.ScriptType.PYDML); double[][] m1 = new double[][] { { 1.0, 2.0 }, { 3.0, 4.0 } }; script.setScriptString("sum1 = sum(m1)").in("m1", m1).out("sum1");
