Repository: incubator-systemml Updated Branches: refs/heads/master b0a2022e9 -> ccdb61fd6
[SYSTEMML-1511] Tab completion for scripts using MLContext Generate MLContext classes at build time to allow tab completion and function execution from the Spark Shell for existing scripts in the main jar file. The generated script classes are subclasses of Script. The results of functions that return multiple values are encapsulated in output classes. This avoids the generic type erasure issue associated with returning results as tuples. Additionally generate methods to return function documentation and full function body. Closes #473, Closes #194. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/ccdb61fd Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/ccdb61fd Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/ccdb61fd Branch: refs/heads/master Commit: ccdb61fd60917c7e20b01c83d72f6a3a829f2a63 Parents: b0a2022 Author: Deron Eriksson <de...@us.ibm.com> Authored: Mon May 8 11:48:05 2017 -0700 Committer: Deron Eriksson <de...@us.ibm.com> Committed: Mon May 8 11:48:05 2017 -0700 ---------------------------------------------------------------------- pom.xml | 19 +- .../java/org/apache/sysml/api/DMLScript.java | 4 + .../org/apache/sysml/api/mlcontext/Script.java | 12 + .../org/apache/sysml/parser/DataExpression.java | 7 + .../parser/common/CommonSyntacticValidator.java | 17 +- .../utils/GenerateClassesForMLContext.java | 1156 ++++++++++++++++++ 6 files changed, 1212 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ccdb61fd/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 4cca9a0..d9ce94e 100644 --- a/pom.xml +++ b/pom.xml @@ -633,9 +633,10 @@ <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>exec-maven-plugin</artifactId> - <version>1.5.0</version> + <version>1.6.0</version> <executions> <execution> + <id>install-dependency-for-integration-tests</id> <phase>prepare-package</phase> <goals> <goal>java</goal> @@ -648,6 +649,20 @@ </arguments> </configuration> </execution> + <execution> + <id>generate-classes-for-mlcontext</id> + <phase>prepare-package</phase> + <goals> + <goal>java</goal> + </goals> + <configuration> + <mainClass>org.apache.sysml.utils.GenerateClassesForMLContext</mainClass> + <arguments> + <arguments>scripts</arguments> + <arguments>target/classes</arguments> + </arguments> + </configuration> + </execution> </executions> </plugin> </plugins> @@ -1044,7 +1059,7 @@ <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>exec-maven-plugin</artifactId> - <version>1.5.0</version> + <version>1.6.0</version> <executions> <execution> <id>generate-python-package-1</id> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ccdb61fd/src/main/java/org/apache/sysml/api/DMLScript.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java index 7988ff0..ea36b12 100644 --- a/src/main/java/org/apache/sysml/api/DMLScript.java +++ b/src/main/java/org/apache/sysml/api/DMLScript.java @@ -178,6 +178,10 @@ public class DMLScript public static boolean _suppressPrint2Stdout = false; // flag that indicates whether or not to suppress any prints to stdout public static boolean USE_LOCAL_SPARK_CONFIG = false; //set default local spark configuration - used for local testing public static boolean _activeAM = false; + /** + * If true, allow DMLProgram to be generated while not halting due to validation errors/warnings + */ + public static boolean VALIDATOR_IGNORE_ISSUES = false; public static String _uuid = IDHandler.createDistributedUniqueID(); private static final Log LOG = LogFactory.getLog(DMLScript.class.getName()); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ccdb61fd/src/main/java/org/apache/sysml/api/mlcontext/Script.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/Script.java b/src/main/java/org/apache/sysml/api/mlcontext/Script.java index 54903b7..27ef5af 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/Script.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/Script.java @@ -678,4 +678,16 @@ public class Script { return this; } + /** + * Execute the script and return the results as an MLResults object. + * + * @return results as an MLResults object + */ + public MLResults execute() { + MLContext ml = MLContext.getActiveMLContext(); + if (ml == null) { + throw new MLContextException("No MLContext object exists. Please create one."); + } + return ml.execute(this); + } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ccdb61fd/src/main/java/org/apache/sysml/parser/DataExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DataExpression.java b/src/main/java/org/apache/sysml/parser/DataExpression.java index 9e8dca9..bb10af4 100644 --- a/src/main/java/org/apache/sysml/parser/DataExpression.java +++ b/src/main/java/org/apache/sysml/parser/DataExpression.java @@ -29,6 +29,7 @@ import java.util.Map.Entry; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.sysml.api.DMLScript; import org.apache.sysml.conf.CompilerConfig.ConfigType; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.hops.DataGenOp; @@ -297,6 +298,9 @@ public class DataExpression extends DataIdentifier public void addRandExprParam(String paramName, Expression paramValue) throws LanguageException { + if (DMLScript.VALIDATOR_IGNORE_ISSUES && (paramValue == null)) { + return; + } // check name is valid boolean found = false; if (paramName != null ){ @@ -484,6 +488,9 @@ public class DataExpression extends DataIdentifier } public void addVarParam(String name, Expression value){ + if (DMLScript.VALIDATOR_IGNORE_ISSUES && (value == null)) { + return; + } _varParams.put(name, value); // if required, initialize values http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ccdb61fd/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java index f76694e..08b4ffa 100644 --- a/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/common/CommonSyntacticValidator.java @@ -29,6 +29,7 @@ import java.util.regex.Pattern; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.Token; +import org.apache.sysml.api.DMLScript; import org.apache.sysml.parser.AssignmentStatement; import org.apache.sysml.parser.BinaryExpression; import org.apache.sysml.parser.BooleanExpression; @@ -88,7 +89,9 @@ public abstract class CommonSyntacticValidator { } protected void notifyErrorListeners(String message, Token op) { - errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message); + if (!DMLScript.VALIDATOR_IGNORE_ISSUES) { + errorListener.validationError(op.getLine(), op.getCharPositionInLine(), message); + } } protected void raiseWarning(String message, Token op) { @@ -487,6 +490,18 @@ public abstract class CommonSyntacticValidator { protected void setPrintStatement(ParserRuleContext ctx, String functionName, ArrayList<ParameterExpression> paramExpression, StatementInfo thisinfo) { + if (DMLScript.VALIDATOR_IGNORE_ISSUES == true) { // create dummy print statement + try { + int line = ctx.start.getLine(); + int col = ctx.start.getCharPositionInLine(); + ArrayList<Expression> expList = new ArrayList<Expression>(); + thisinfo.stmt = new PrintStatement(functionName, expList, line, col, line, col); + setFileLineColumn(thisinfo.stmt, ctx); + } catch (LanguageException e) { + e.printStackTrace(); + } + return; + } int numParams = paramExpression.size(); if (numParams == 0) { notifyErrorListeners(functionName + "() must have more than 0 parameters", ctx.start); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/ccdb61fd/src/main/java/org/apache/sysml/utils/GenerateClassesForMLContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/GenerateClassesForMLContext.java b/src/main/java/org/apache/sysml/utils/GenerateClassesForMLContext.java new file mode 100644 index 0000000..f6002b0 --- /dev/null +++ b/src/main/java/org/apache/sysml/utils/GenerateClassesForMLContext.java @@ -0,0 +1,1156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysml.utils; + +import java.io.File; +import java.io.FileFilter; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringEscapeUtils; +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.mlcontext.MLContext; +import org.apache.sysml.api.mlcontext.MLResults; +import org.apache.sysml.api.mlcontext.Script; +import org.apache.sysml.api.mlcontext.ScriptExecutor; +import org.apache.sysml.parser.DMLProgram; +import org.apache.sysml.parser.DataIdentifier; +import org.apache.sysml.parser.Expression.DataType; +import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.parser.FunctionStatement; +import org.apache.sysml.parser.FunctionStatementBlock; +import org.apache.sysml.parser.LanguageException; +import org.apache.sysml.parser.Statement; + +import javassist.CannotCompileException; +import javassist.ClassPool; +import javassist.CtClass; +import javassist.CtConstructor; +import javassist.CtField; +import javassist.CtMethod; +import javassist.CtNewConstructor; +import javassist.CtNewMethod; +import javassist.Modifier; +import javassist.NotFoundException; + +/** + * Automatically generate classes and methods for interaction with DML scripts + * and functions through the MLContext API. + * + */ +public class GenerateClassesForMLContext { + + public static final String SOURCE = "scripts"; + public static final String DESTINATION = "target/classes"; + public static final String BASE_DEST_PACKAGE = "org.apache.sysml"; + public static final String CONVENIENCE_BASE_DEST_PACKAGE = "org.apache.sysml.api.mlcontext.convenience"; + public static final String PATH_TO_MLCONTEXT_CLASS = "org/apache/sysml/api/mlcontext/MLContext.class"; + public static final String PATH_TO_MLRESULTS_CLASS = "org/apache/sysml/api/mlcontext/MLResults.class"; + public static final String PATH_TO_SCRIPT_CLASS = "org/apache/sysml/api/mlcontext/Script.class"; + public static final String PATH_TO_SCRIPTTYPE_CLASS = "org/apache/sysml/api/mlcontext/ScriptType.class"; + public static final String PATH_TO_MATRIX_CLASS = "org/apache/sysml/api/mlcontext/Matrix.class"; + public static final String PATH_TO_FRAME_CLASS = "org/apache/sysml/api/mlcontext/Frame.class"; + + public static String source = SOURCE; + public static String destination = DESTINATION; + public static boolean skipStagingDir = true; + public static boolean skipPerfTestDir = true; + public static boolean skipObsoleteDir = true; + public static boolean skipTestDir = true; + + public static void main(String[] args) throws Throwable { + if (args.length == 2) { + source = args[0]; + destination = args[1]; + } else if (args.length == 1) { + source = args[0]; + } + try { + DMLScript.VALIDATOR_IGNORE_ISSUES = true; + System.out.println("************************************"); + System.out.println("**** MLContext Class Generation ****"); + System.out.println("************************************"); + System.out.println("Source: " + source); + System.out.println("Destination: " + destination); + makeCtClasses(); + recurseDirectoriesForClassGeneration(source); + String fullDirClassName = recurseDirectoriesForConvenienceClassGeneration(source); + addConvenienceMethodsToMLContext(source, fullDirClassName); + } finally { + DMLScript.VALIDATOR_IGNORE_ISSUES = false; + } + } + + /** + * Create compile-time classes required for later class generation. + */ + public static void makeCtClasses() { + try { + ClassPool pool = ClassPool.getDefault(); + pool.makeClass(new FileInputStream(new File(destination + File.separator + PATH_TO_MLCONTEXT_CLASS))); + pool.makeClass(new FileInputStream(new File(destination + File.separator + PATH_TO_MLRESULTS_CLASS))); + pool.makeClass(new FileInputStream(new File(destination + File.separator + PATH_TO_SCRIPT_CLASS))); + pool.makeClass(new FileInputStream(new File(destination + File.separator + PATH_TO_SCRIPTTYPE_CLASS))); + pool.makeClass(new FileInputStream(new File(destination + File.separator + PATH_TO_MATRIX_CLASS))); + pool.makeClass(new FileInputStream(new File(destination + File.separator + PATH_TO_FRAME_CLASS))); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (RuntimeException e) { + e.printStackTrace(); + } + } + + /** + * Add methods to MLContext to allow tab-completion to folders/packages + * (such as {@code ml.scripts()} and {@code ml.nn()}). + * + * @param source + * path to source directory (typically, the scripts directory) + * @param fullDirClassName + * the full name of the class representing the source (scripts) + * directory + */ + public static void addConvenienceMethodsToMLContext(String source, String fullDirClassName) { + try { + ClassPool pool = ClassPool.getDefault(); + CtClass ctMLContext = pool.get(MLContext.class.getName()); + + CtClass dirClass = pool.get(fullDirClassName); + String methodName = convertFullClassNameToConvenienceMethodName(fullDirClassName); + System.out.println("Adding " + methodName + "() to " + ctMLContext.getName()); + + String methodBody = "{ " + fullDirClassName + " z = new " + fullDirClassName + "(); return z; }"; + CtMethod ctMethod = CtNewMethod.make(Modifier.PUBLIC, dirClass, methodName, null, null, methodBody, + ctMLContext); + ctMLContext.addMethod(ctMethod); + + addPackageConvenienceMethodsToMLContext(source, ctMLContext); + + ctMLContext.writeFile(destination); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (RuntimeException e) { + e.printStackTrace(); + } catch (NotFoundException e) { + e.printStackTrace(); + } catch (CannotCompileException e) { + e.printStackTrace(); + } + } + + /** + * Add methods to MLContext to allow tab-completion to packages contained + * within the source directory (such as {@code ml.nn()}). + * + * @param dirPath + * path to source directory (typically, the scripts directory) + * @param ctMLContext + * javassist compile-time class representation of MLContext + */ + public static void addPackageConvenienceMethodsToMLContext(String dirPath, CtClass ctMLContext) { + + try { + if (!SOURCE.equalsIgnoreCase(dirPath)) { + return; + } + File dir = new File(dirPath); + File[] subdirs = dir.listFiles(new FileFilter() { + @Override + public boolean accept(File f) { + return f.isDirectory(); + } + }); + for (File subdir : subdirs) { + String subDirPath = dirPath + File.separator + subdir.getName(); + if (skipDir(subdir, false)) { + continue; + } + + String fullSubDirClassName = dirPathToFullDirClassName(subDirPath); + + ClassPool pool = ClassPool.getDefault(); + CtClass subDirClass = pool.get(fullSubDirClassName); + String subDirName = subdir.getName(); + subDirName = subDirName.replaceAll("-", "_"); + subDirName = subDirName.toLowerCase(); + + System.out.println("Adding " + subDirName + "() to " + ctMLContext.getName()); + + String methodBody = "{ " + fullSubDirClassName + " z = new " + fullSubDirClassName + "(); return z; }"; + CtMethod ctMethod = CtNewMethod.make(Modifier.PUBLIC, subDirClass, subDirName, null, null, methodBody, + ctMLContext); + ctMLContext.addMethod(ctMethod); + + } + } catch (NotFoundException e) { + e.printStackTrace(); + } catch (CannotCompileException e) { + e.printStackTrace(); + } + + } + + /** + * Convert the full name of a class representing a directory to a method + * name. + * + * @param fullDirClassName + * the full name of the class representing a directory + * @return method name + */ + public static String convertFullClassNameToConvenienceMethodName(String fullDirClassName) { + String m = fullDirClassName; + m = m.substring(m.lastIndexOf(".") + 1); + m = m.toLowerCase(); + return m; + } + + /** + * Generate convenience classes recursively. This allows for code such as + * {@code ml.scripts.algorithms...}. + * + * @param dirPath + * path to directory + * @return the full name of the class representing the dirPath directory + */ + public static String recurseDirectoriesForConvenienceClassGeneration(String dirPath) { + try { + File dir = new File(dirPath); + + String fullDirClassName = dirPathToFullDirClassName(dirPath); + System.out.println("Generating Class: " + fullDirClassName); + + ClassPool pool = ClassPool.getDefault(); + CtClass ctDir = pool.makeClass(fullDirClassName); + + File[] subdirs = dir.listFiles(new FileFilter() { + @Override + public boolean accept(File f) { + return f.isDirectory(); + } + }); + for (File subdir : subdirs) { + String subDirPath = dirPath + File.separator + subdir.getName(); + if (skipDir(subdir, false)) { + continue; + } + String fullSubDirClassName = recurseDirectoriesForConvenienceClassGeneration(subDirPath); + + CtClass subDirClass = pool.get(fullSubDirClassName); + String subDirName = subdir.getName(); + subDirName = subDirName.replaceAll("-", "_"); + subDirName = subDirName.toLowerCase(); + + System.out.println("Adding " + subDirName + "() to " + fullDirClassName); + + String methodBody = "{ " + fullSubDirClassName + " z = new " + fullSubDirClassName + "(); return z; }"; + CtMethod ctMethod = CtNewMethod.make(Modifier.PUBLIC, subDirClass, subDirName, null, null, methodBody, + ctDir); + ctDir.addMethod(ctMethod); + + } + + File[] scriptFiles = dir.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return (name.toLowerCase().endsWith(".dml") || name.toLowerCase().endsWith(".pydml")); + } + }); + for (File scriptFile : scriptFiles) { + String scriptFilePath = scriptFile.getPath(); + String fullScriptClassName = BASE_DEST_PACKAGE + "." + + scriptFilePathToFullClassNameNoBase(scriptFilePath); + CtClass scriptClass = pool.get(fullScriptClassName); + String methodName = scriptFilePathToSimpleClassName(scriptFilePath); + String methodBody = "{ " + fullScriptClassName + " z = new " + fullScriptClassName + "(); return z; }"; + CtMethod ctMethod = CtNewMethod.make(Modifier.PUBLIC, scriptClass, methodName, null, null, methodBody, + ctDir); + ctDir.addMethod(ctMethod); + + } + + ctDir.writeFile(destination); + + return fullDirClassName; + } catch (RuntimeException e) { + e.printStackTrace(); + } catch (CannotCompileException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (NotFoundException e) { + e.printStackTrace(); + } + return null; + } + + /** + * Convert a directory path to a full class name for a convenience class. + * + * @param dirPath + * path to directory + * @return the full name of the class representing the dirPath directory + */ + public static String dirPathToFullDirClassName(String dirPath) { + if (!dirPath.contains(File.separator)) { + String c = dirPath; + c = c.replace("-", "_"); + c = c.substring(0, 1).toUpperCase() + c.substring(1); + c = CONVENIENCE_BASE_DEST_PACKAGE + "." + c; + return c; + } + + String p = dirPath; + p = p.substring(0, p.lastIndexOf(File.separator)); + p = p.replace("-", "_"); + p = p.replace(File.separator, "."); + p = p.toLowerCase(); + + String c = dirPath; + c = c.substring(c.lastIndexOf(File.separator) + 1); + c = c.replace("-", "_"); + c = c.substring(0, 1).toUpperCase() + c.substring(1); + + return CONVENIENCE_BASE_DEST_PACKAGE + "." + p + "." + c; + } + + /** + * Whether or not the directory (and subdirectories of the directory) should + * be skipped. + * + * @param dir + * path to directory to check + * @param displayMessage + * if {@code true}, display skip information to standard output + * @return {@code true} if the directory should be skipped, {@code false} + * otherwise + */ + public static boolean skipDir(File dir, boolean displayMessage) { + if ("staging".equalsIgnoreCase(dir.getName()) && skipStagingDir) { + if (displayMessage) { + System.out.println("Skipping staging directory: " + dir.getPath()); + } + return true; + } + if ("perftest".equalsIgnoreCase(dir.getName()) && skipPerfTestDir) { + if (displayMessage) { + System.out.println("Skipping perftest directory: " + dir.getPath()); + } + return true; + } + if ("obsolete".equalsIgnoreCase(dir.getName()) && skipObsoleteDir) { + if (displayMessage) { + System.out.println("Skipping obsolete directory: " + dir.getPath()); + } + return true; + } + if ("test".equalsIgnoreCase(dir.getName()) && skipTestDir) { + if (displayMessage) { + System.out.println("Skipping test directory: " + dir.getPath()); + } + return true; + } + return false; + } + + /** + * Recursively traverse the directories to create classes representing the + * script files. + * + * @param dirPath + * path to directory + */ + public static void recurseDirectoriesForClassGeneration(String dirPath) { + File dir = new File(dirPath); + + iterateScriptFilesInDirectory(dir); + + File[] subdirs = dir.listFiles(new FileFilter() { + @Override + public boolean accept(File f) { + return f.isDirectory(); + } + }); + for (File subdir : subdirs) { + String subdirpath = dirPath + File.separator + subdir.getName(); + if (skipDir(subdir, true)) { + continue; + } + recurseDirectoriesForClassGeneration(subdirpath); + } + } + + /** + * Iterate through the script files in a directory and create a class for + * each script file. + * + * @param dir + * the directory to iterate through + */ + public static void iterateScriptFilesInDirectory(File dir) { + File[] scriptFiles = dir.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return (name.toLowerCase().endsWith(".dml") || name.toLowerCase().endsWith(".pydml")); + } + }); + for (File scriptFile : scriptFiles) { + String scriptFilePath = scriptFile.getPath(); + createScriptClass(scriptFilePath); + } + } + + /** + * Obtain the relative package for a script file. For example, + * {@code scripts/algorithms/LinearRegCG.dml} resolves to + * {@code scripts.algorithms}. + * + * @param scriptFilePath + * the path to a script file + * @return the relative package for a script file + */ + public static String scriptFilePathToPackageNoBase(String scriptFilePath) { + String p = scriptFilePath; + p = p.substring(0, p.lastIndexOf(File.separator)); + p = p.replace("-", "_"); + p = p.replace(File.separator, "."); + p = p.toLowerCase(); + return p; + } + + /** + * Obtain the simple class name for a script file. For example, + * {@code scripts/algorithms/LinearRegCG} resolves to {@code LinearRegCG}. + * + * @param scriptFilePath + * the path to a script file + * @return the simple class name for a script file + */ + public static String scriptFilePathToSimpleClassName(String scriptFilePath) { + String c = scriptFilePath; + c = c.substring(c.lastIndexOf(File.separator) + 1); + c = c.replace("-", "_"); + c = c.substring(0, 1).toUpperCase() + c.substring(1); + c = c.substring(0, c.indexOf(".")); + return c; + } + + /** + * Obtain the relative full class name for a script file. For example, + * {@code scripts/algorithms/LinearRegCG.dml} resolves to + * {@code scripts.algorithms.LinearRegCG}. + * + * @param scriptFilePath + * the path to a script file + * @return the relative full class name for a script file + */ + public static String scriptFilePathToFullClassNameNoBase(String scriptFilePath) { + String p = scriptFilePathToPackageNoBase(scriptFilePath); + String c = scriptFilePathToSimpleClassName(scriptFilePath); + return p + "." + c; + } + + /** + * Convert a script file to a Java class that extends the MLContext API's + * Script class. + * + * @param scriptFilePath + * the path to a script file + */ + public static void createScriptClass(String scriptFilePath) { + try { + String fullScriptClassName = BASE_DEST_PACKAGE + "." + scriptFilePathToFullClassNameNoBase(scriptFilePath); + System.out.println("Generating Class: " + fullScriptClassName); + ClassPool pool = ClassPool.getDefault(); + CtClass ctNewScript = pool.makeClass(fullScriptClassName); + + CtClass ctScript = pool.get(Script.class.getName()); + ctNewScript.setSuperclass(ctScript); + + CtConstructor ctCon = new CtConstructor(null, ctNewScript); + ctCon.setBody(scriptConstructorBody(scriptFilePath)); + ctNewScript.addConstructor(ctCon); + + addFunctionMethods(scriptFilePath, ctNewScript); + + ctNewScript.writeFile(destination); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (RuntimeException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } catch (CannotCompileException e) { + e.printStackTrace(); + } catch (NotFoundException e) { + e.printStackTrace(); + } + + } + + /** + * Create a DMLProgram from a script file. + * + * @param scriptFilePath + * the path to a script file + * @return the DMLProgram generated by the script file + */ + public static DMLProgram dmlProgramFromScriptFilePath(String scriptFilePath) { + String scriptString = fileToString(scriptFilePath); + Script script = new Script(scriptString); + ScriptExecutor se = new ScriptExecutor() { + @Override + public MLResults execute(Script script) { + setup(script); + parseScript(); + return null; + } + }; + se.execute(script); + DMLProgram dmlProgram = se.getDmlProgram(); + return dmlProgram; + } + + /** + * Add methods to a derived script class to allow invocation of script + * functions. + * + * @param scriptFilePath + * the path to a script file + * @param ctNewScript + * the javassist compile-time class representation of a script + */ + public static void addFunctionMethods(String scriptFilePath, CtClass ctNewScript) { + try { + DMLProgram dmlProgram = dmlProgramFromScriptFilePath(scriptFilePath); + if (dmlProgram == null) { + System.out.println("Could not generate DML Program for: " + scriptFilePath); + return; + } + Map<String, FunctionStatementBlock> defaultNsFsbsMap = dmlProgram + .getFunctionStatementBlocks(DMLProgram.DEFAULT_NAMESPACE); + List<FunctionStatementBlock> fsbs = new ArrayList<FunctionStatementBlock>(); + fsbs.addAll(defaultNsFsbsMap.values()); + for (FunctionStatementBlock fsb : fsbs) { + ArrayList<Statement> sts = fsb.getStatements(); + for (Statement st : sts) { + if (!(st instanceof FunctionStatement)) { + continue; + } + FunctionStatement fs = (FunctionStatement) st; + + String dmlFunctionCall = generateDmlFunctionCall(scriptFilePath, fs); + String functionCallMethod = generateFunctionCallMethod(scriptFilePath, fs, dmlFunctionCall); + + CtMethod m = CtNewMethod.make(functionCallMethod, ctNewScript); + ctNewScript.addMethod(m); + + addDescriptionFunctionCallMethod(fs, scriptFilePath, ctNewScript, false); + addDescriptionFunctionCallMethod(fs, scriptFilePath, ctNewScript, true); + } + } + } catch (LanguageException e) { + System.out.println("Could not add function methods for " + ctNewScript.getName()); + } catch (CannotCompileException e) { + System.out.println("Could not add function methods for " + ctNewScript.getName()); + } catch (RuntimeException e) { + System.out.println("Could not add function methods for " + ctNewScript.getName()); + } + } + + /** + * Create a method that returns either: (1) the full function body, or (2) + * the function body up to the end of the documentation comment for the + * function. If (1) is generated, the method name will be followed + * "__source". If (2) is generated, the method name will be followed by + * "__docs". If (2) is generated but no end of documentation comment is + * detected, the full function body will be displayed. + * + * @param fs + * a SystemML function statement + * @param scriptFilePath + * the path to a script file + * @param ctNewScript + * the javassist compile-time class representation of a script + * @param full + * if {@code true}, create method to return full function body; + * if {@code false}, create method to return the function body up + * to the end of the documentation comment + */ + public static void addDescriptionFunctionCallMethod(FunctionStatement fs, String scriptFilePath, + CtClass ctNewScript, boolean full) { + + try { + int bl = fs.getBeginLine(); + int el = fs.getEndLine(); + File f = new File(scriptFilePath); + List<String> lines = FileUtils.readLines(f); + + int end = el; + if (!full) { + for (int i = bl - 1; i < el; i++) { + String line = lines.get(i); + if (line.contains("*/")) { + end = i + 1; + break; + } + } + } + List<String> sub = lines.subList(bl - 1, end); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < sub.size(); i++) { + String line = sub.get(i); + String escapeLine = StringEscapeUtils.escapeJava(line); + sb.append(escapeLine); + sb.append("\\n"); + } + + String functionString = sb.toString(); + String docFunctionCallMethod = generateDescriptionFunctionCallMethod(fs, functionString, full); + CtMethod m = CtNewMethod.make(docFunctionCallMethod, ctNewScript); + ctNewScript.addMethod(m); + + } catch (IOException e) { + e.printStackTrace(); + } catch (CannotCompileException e) { + e.printStackTrace(); + } + + } + + /** + * Generate method for returning (1) the full function body, or (2) the + * function body up to the end of the documentation comment. (1) will have + * "__source" appended to the end of the function name. (2) will have + * "__docs" appended to the end of the function name. + * + * @param fs + * a SystemML function statement + * @param functionString + * either the full function body or the function body up to the + * end of the documentation comment + * @param full + * if {@code true}, append "__source" to the end of the function + * name; if {@code false}, append "__docs" to the end of the + * function name + * @return string representation of the function description method + */ + public static String generateDescriptionFunctionCallMethod(FunctionStatement fs, String functionString, + boolean full) { + StringBuilder sb = new StringBuilder(); + sb.append("public String "); + sb.append(fs.getName()); + if (full) { + sb.append("__source"); + } else { + sb.append("__docs"); + } + sb.append("() {\n"); + sb.append("String docString = \"" + functionString + "\";\n"); + sb.append("return docString;\n"); + sb.append("}\n"); + return sb.toString(); + } + + /** + * Obtain a string representation of a parameter type, where a Matrix or + * Frame is represented by its full class name. + * + * @param param + * the function parameter + * @return string representation of a parameter type + */ + public static String getParamTypeAsString(DataIdentifier param) { + DataType dt = param.getDataType(); + ValueType vt = param.getValueType(); + if ((dt == DataType.SCALAR) && (vt == ValueType.INT)) { + return "long"; + } else if ((dt == DataType.SCALAR) && (vt == ValueType.DOUBLE)) { + return "double"; + } else if ((dt == DataType.SCALAR) && (vt == ValueType.BOOLEAN)) { + return "boolean"; + } else if ((dt == DataType.SCALAR) && (vt == ValueType.STRING)) { + return "String"; + } else if (dt == DataType.MATRIX) { + return "org.apache.sysml.api.mlcontext.Matrix"; + } else if (dt == DataType.FRAME) { + return "org.apache.sysml.api.mlcontext.Frame"; + } + return null; + } + + /** + * Obtain a string representation of a parameter type, where a Matrix or + * Frame is represented by its simple class name. + * + * @param param + * the function parameter + * @return string representation of a parameter type + */ + public static String getSimpleParamTypeAsString(DataIdentifier param) { + DataType dt = param.getDataType(); + ValueType vt = param.getValueType(); + if ((dt == DataType.SCALAR) && (vt == ValueType.INT)) { + return "long"; + } else if ((dt == DataType.SCALAR) && (vt == ValueType.DOUBLE)) { + return "double"; + } else if ((dt == DataType.SCALAR) && (vt == ValueType.BOOLEAN)) { + return "boolean"; + } else if ((dt == DataType.SCALAR) && (vt == ValueType.STRING)) { + return "String"; + } else if (dt == DataType.MATRIX) { + return "Matrix"; + } else if (dt == DataType.FRAME) { + return "Frame"; + } + return null; + } + + /** + * Obtain the full class name for a class that encapsulates the outputs of a + * function + * + * @param scriptFilePath + * the path to a script file + * @param fs + * a SystemML function statement + * @return the full class name for a class that encapsulates the outputs of + * a function + */ + public static String getFullFunctionOutputClassName(String scriptFilePath, FunctionStatement fs) { + String p = scriptFilePath; + p = p.replace("-", "_"); + p = p.replace(File.separator, "."); + p = p.toLowerCase(); + p = p.substring(0, p.lastIndexOf(".")); + + String c = fs.getName(); + c = c.substring(0, 1).toUpperCase() + c.substring(1); + c = c + "_output"; + + String functionOutputClassName = BASE_DEST_PACKAGE + "." + p + "." + c; + return functionOutputClassName; + } + + /** + * Create a class that encapsulates the outputs of a function. + * + * @param scriptFilePath + * the path to a script file + * @param fs + * a SystemML function statement + */ + public static void createFunctionOutputClass(String scriptFilePath, FunctionStatement fs) { + + try { + ArrayList<DataIdentifier> oparams = fs.getOutputParams(); + // Note: if a function returns 1 output, simply output it rather + // than encapsulating it in a function output class + if ((oparams.size() == 0) || (oparams.size() == 1)) { + return; + } + + String fullFunctionOutputClassName = getFullFunctionOutputClassName(scriptFilePath, fs); + System.out.println("Generating Class: " + fullFunctionOutputClassName); + ClassPool pool = ClassPool.getDefault(); + CtClass ctFuncOut = pool.makeClass(fullFunctionOutputClassName); + + // add fields + for (int i = 0; i < oparams.size(); i++) { + DataIdentifier oparam = oparams.get(i); + String type = getParamTypeAsString(oparam); + String name = oparam.getName(); + String fstring = "public " + type + " " + name + ";"; + CtField field = CtField.make(fstring, ctFuncOut); + ctFuncOut.addField(field); + } + + // add constructor + String simpleFuncOutClassName = fullFunctionOutputClassName + .substring(fullFunctionOutputClassName.lastIndexOf(".") + 1); + StringBuilder con = new StringBuilder(); + con.append("public " + simpleFuncOutClassName + "("); + for (int i = 0; i < oparams.size(); i++) { + if (i > 0) { + con.append(", "); + } + DataIdentifier oparam = oparams.get(i); + String type = getParamTypeAsString(oparam); + String name = oparam.getName(); + con.append(type + " " + name); + } + con.append(") {\n"); + for (int i = 0; i < oparams.size(); i++) { + DataIdentifier oparam = oparams.get(i); + String name = oparam.getName(); + con.append("this." + name + "=" + name + ";\n"); + } + con.append("}\n"); + String cstring = con.toString(); + CtConstructor ctCon = CtNewConstructor.make(cstring, ctFuncOut); + ctFuncOut.addConstructor(ctCon); + + // add toString + StringBuilder s = new StringBuilder(); + s.append("public String toString(){\n"); + s.append("StringBuilder sb = new StringBuilder();\n"); + for (int i = 0; i < oparams.size(); i++) { + DataIdentifier oparam = oparams.get(i); + String name = oparam.getName(); + s.append("sb.append(\"" + name + " (" + getSimpleParamTypeAsString(oparam) + "): \" + " + name + + " + \"\\n\");\n"); + } + s.append("String str = sb.toString();\nreturn str;\n"); + s.append("}\n"); + String toStr = s.toString(); + CtMethod toStrMethod = CtNewMethod.make(toStr, ctFuncOut); + ctFuncOut.addMethod(toStrMethod); + + ctFuncOut.writeFile(destination); + + } catch (RuntimeException e) { + e.printStackTrace(); + } catch (CannotCompileException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** + * Obtain method for invoking a script function. + * + * @param scriptFilePath + * the path to a script file + * @param fs + * a SystemML function statement + * @param dmlFunctionCall + * a string representing the invocation of a script function + * @return string representation of a method that performs a function call + */ + public static String generateFunctionCallMethod(String scriptFilePath, FunctionStatement fs, + String dmlFunctionCall) { + + createFunctionOutputClass(scriptFilePath, fs); + + StringBuilder sb = new StringBuilder(); + sb.append("public "); + + // begin return type + ArrayList<DataIdentifier> oparams = fs.getOutputParams(); + if (oparams.size() == 0) { + sb.append("void"); + } else if (oparams.size() == 1) { + // if 1 output, no need to encapsulate it, so return the output + // directly + DataIdentifier oparam = oparams.get(0); + String type = getParamTypeAsString(oparam); + sb.append(type); + } else { + String fullFunctionOutputClassName = getFullFunctionOutputClassName(scriptFilePath, fs); + sb.append(fullFunctionOutputClassName); + } + sb.append(" "); + // end return type + + sb.append(fs.getName()); + sb.append("("); + + ArrayList<DataIdentifier> inputParams = fs.getInputParams(); + for (int i = 0; i < inputParams.size(); i++) { + if (i > 0) { + sb.append(", "); + } + DataIdentifier inputParam = inputParams.get(i); + /* + * Note: Using Object is currently preferrable to using + * datatype/valuetype to explicitly set the input type to + * Integer/Double/Boolean/String since Object allows the automatic + * handling of things such as automatic conversions from longs to + * ints. + */ + sb.append("Object "); + sb.append(inputParam.getName()); + } + + sb.append(") {\n"); + sb.append("String scriptString = \"" + dmlFunctionCall + "\";\n"); + sb.append( + "org.apache.sysml.api.mlcontext.Script script = new org.apache.sysml.api.mlcontext.Script(scriptString);\n"); + + if ((inputParams.size() > 0) || (oparams.size() > 0)) { + sb.append("script"); + } + for (int i = 0; i < inputParams.size(); i++) { + DataIdentifier inputParam = inputParams.get(i); + String name = inputParam.getName(); + sb.append(".in(\"" + name + "\", " + name + ")"); + } + for (int i = 0; i < oparams.size(); i++) { + DataIdentifier outputParam = oparams.get(i); + String name = outputParam.getName(); + sb.append(".out(\"" + name + "\")"); + } + if ((inputParams.size() > 0) || (oparams.size() > 0)) { + sb.append(";\n"); + } + + sb.append("org.apache.sysml.api.mlcontext.MLResults results = script.execute();\n"); + + if (oparams.size() == 0) { + sb.append("return;\n"); + } else if (oparams.size() == 1) { + DataIdentifier o = oparams.get(0); + DataType dt = o.getDataType(); + ValueType vt = o.getValueType(); + if ((dt == DataType.SCALAR) && (vt == ValueType.INT)) { + sb.append("long res = results.getLong(\"" + o.getName() + "\");\nreturn res;\n"); + } else if ((dt == DataType.SCALAR) && (vt == ValueType.DOUBLE)) { + sb.append("double res = results.getDouble(\"" + o.getName() + "\");\nreturn res;\n"); + } else if ((dt == DataType.SCALAR) && (vt == ValueType.BOOLEAN)) { + sb.append("boolean res = results.getBoolean(\"" + o.getName() + "\");\nreturn res;\n"); + } else if ((dt == DataType.SCALAR) && (vt == ValueType.STRING)) { + sb.append("String res = results.getString(\"" + o.getName() + "\");\nreturn res;\n"); + } else if (dt == DataType.MATRIX) { + sb.append("org.apache.sysml.api.mlcontext.Matrix res = results.getMatrix(\"" + o.getName() + + "\");\nreturn res;\n"); + } else if (dt == DataType.FRAME) { + sb.append("org.apache.sysml.api.mlcontext.Frame res = results.getFrame(\"" + o.getName() + + "\");\nreturn res;\n"); + } + } else { + + for (int i = 0; i < oparams.size(); i++) { + DataIdentifier outputParam = oparams.get(i); + String name = outputParam.getName().toLowerCase(); + String type = getParamTypeAsString(outputParam); + DataType dt = outputParam.getDataType(); + ValueType vt = outputParam.getValueType(); + if ((dt == DataType.SCALAR) && (vt == ValueType.INT)) { + sb.append(type + " " + name + " = results.getLong(\"" + outputParam.getName() + "\");\n"); + } else if ((dt == DataType.SCALAR) && (vt == ValueType.DOUBLE)) { + sb.append(type + " " + name + " = results.getDouble(\"" + outputParam.getName() + "\");\n"); + } else if ((dt == DataType.SCALAR) && (vt == ValueType.BOOLEAN)) { + sb.append(type + " " + name + " = results.getBoolean(\"" + outputParam.getName() + "\");\n"); + } else if ((dt == DataType.SCALAR) && (vt == ValueType.STRING)) { + sb.append(type + " " + name + " = results.getString(\"" + outputParam.getName() + "\");\n"); + } else if (dt == DataType.MATRIX) { + sb.append(type + " " + name + " = results.getMatrix(\"" + outputParam.getName() + "\");\n"); + } else if (dt == DataType.FRAME) { + sb.append(type + " " + name + " = results.getFrame(\"" + outputParam.getName() + "\");\n"); + } + } + String ffocn = getFullFunctionOutputClassName(scriptFilePath, fs); + sb.append(ffocn + " res = new " + ffocn + "("); + for (int i = 0; i < oparams.size(); i++) { + if (i > 0) { + sb.append(", "); + } + DataIdentifier outputParam = oparams.get(i); + String name = outputParam.getName().toLowerCase(); + sb.append(name); + } + sb.append(");\nreturn res;\n"); + } + + sb.append("}\n"); + return sb.toString(); + } + + /** + * Obtain method for invoking a script function and returning the results as + * an MLResults object. Currently this method is not used. + * + * @param scriptFilePath + * the path to a script file + * @param fs + * a SystemML function statement + * @param dmlFunctionCall + * a string representing the invocation of a script function + * @return string representation of a method that performs a function call + */ + public static String generateFunctionCallMethodMLResults(String scriptFilePath, FunctionStatement fs, + String dmlFunctionCall) { + StringBuilder sb = new StringBuilder(); + + sb.append("public org.apache.sysml.api.mlcontext.MLResults "); + sb.append(fs.getName()); + sb.append("("); + + ArrayList<DataIdentifier> inputParams = fs.getInputParams(); + for (int i = 0; i < inputParams.size(); i++) { + if (i > 0) { + sb.append(", "); + } + DataIdentifier inputParam = inputParams.get(i); + /* + * Note: Using Object is currently preferrable to using + * datatype/valuetype to explicitly set the input type to + * Integer/Double/Boolean/String since Object allows the automatic + * handling of things such as automatic conversions from longs to + * ints. + */ + sb.append("Object "); + sb.append(inputParam.getName()); + } + + sb.append(") {\n"); + sb.append("String scriptString = \"" + dmlFunctionCall + "\";\n"); + sb.append( + "org.apache.sysml.api.mlcontext.Script script = new org.apache.sysml.api.mlcontext.Script(scriptString);\n"); + + ArrayList<DataIdentifier> outputParams = fs.getOutputParams(); + if ((inputParams.size() > 0) || (outputParams.size() > 0)) { + sb.append("script"); + } + for (int i = 0; i < inputParams.size(); i++) { + DataIdentifier inputParam = inputParams.get(i); + String name = inputParam.getName(); + sb.append(".in(\"" + name + "\", " + name + ")"); + } + for (int i = 0; i < outputParams.size(); i++) { + DataIdentifier outputParam = outputParams.get(i); + String name = outputParam.getName(); + sb.append(".out(\"" + name + "\")"); + } + if ((inputParams.size() > 0) || (outputParams.size() > 0)) { + sb.append(";\n"); + } + + sb.append("org.apache.sysml.api.mlcontext.MLResults results = script.execute();\n"); + sb.append("return results;\n"); + sb.append("}\n"); + return sb.toString(); + } + + /** + * Obtain the DML representing a function invocation. + * + * @param scriptFilePath + * the path to a script file + * @param fs + * a SystemML function statement + * @return string representation of a DML function invocation + */ + public static String generateDmlFunctionCall(String scriptFilePath, FunctionStatement fs) { + StringBuilder sb = new StringBuilder(); + sb.append("source('" + scriptFilePath + "') as mlcontextns;"); + + ArrayList<DataIdentifier> outputParams = fs.getOutputParams(); + if (outputParams.size() == 0) { + sb.append("mlcontextns::"); + } + if (outputParams.size() == 1) { + DataIdentifier outputParam = outputParams.get(0); + sb.append(outputParam.getName()); + sb.append(" = mlcontextns::"); + } else if (outputParams.size() > 1) { + sb.append("["); + for (int i = 0; i < outputParams.size(); i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(outputParams.get(i).getName()); + } + sb.append("] = mlcontextns::"); + } + sb.append(fs.getName()); + sb.append("("); + ArrayList<DataIdentifier> inputParams = fs.getInputParams(); + for (int i = 0; i < inputParams.size(); i++) { + if (i > 0) { + sb.append(", "); + } + DataIdentifier inputParam = inputParams.get(i); + sb.append(inputParam.getName()); + } + sb.append(");"); + return sb.toString(); + } + + /** + * Obtain the content of a file as a string. + * + * @param filePath + * the path to a file + * @return the file content as a string + */ + public static String fileToString(String filePath) { + try { + File f = new File(filePath); + FileReader fr = new FileReader(f); + StringBuilder sb = new StringBuilder(); + int n; + char[] charArray = new char[1024]; + while ((n = fr.read(charArray)) > 0) { + sb.append(charArray, 0, n); + } + fr.close(); + return sb.toString(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + /** + * Obtain a constructor body for a Script subclass that sets the + * scriptString based on the content of a script file. + * + * @param scriptFilePath + * the path to a script file + * @return constructor body for a Script subclass that sets the scriptString + * based on the content of a script file + */ + public static String scriptConstructorBody(String scriptFilePath) { + StringBuilder sb = new StringBuilder(); + sb.append("{"); + sb.append("String scriptFilePath = \"" + scriptFilePath + "\";"); + sb.append( + "java.io.InputStream is = org.apache.sysml.api.mlcontext.Script.class.getResourceAsStream(\"/\"+scriptFilePath);"); + sb.append("java.io.InputStreamReader isr = new java.io.InputStreamReader(is);"); + sb.append("int n;"); + sb.append("char[] charArray = new char[1024];"); + sb.append("StringBuilder s = new StringBuilder();"); + sb.append("try {"); + sb.append(" while ((n = isr.read(charArray)) > 0) {"); + sb.append(" s.append(charArray, 0, n);"); + sb.append(" }"); + sb.append("} catch (java.io.IOException e) {"); + sb.append(" e.printStackTrace();"); + sb.append("}"); + sb.append("setScriptString(s.toString());"); + sb.append("}"); + return sb.toString(); + } + +}
