Repository: incubator-systemml Updated Branches: refs/heads/master 17aac2108 -> 71733fd1f
[SYSTEMML-688] Update dml file packaging in artifacts Move dml to scripts dir for main, in-memory, sources, and standalone jars. Include algorithms, datagen, and utils directories. Exclude obsolete algorithms, perftest, and staging. Update distrib.xml and standalone.xml to reflect similar packaging. Update ScriptUtils.scala and LogisticRegression.scala for these changes. Closes #167. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/71733fd1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/71733fd1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/71733fd1 Branch: refs/heads/master Commit: 71733fd1fb13d1c9ddb7c27da6f935547a3bc8ef Parents: 17aac21 Author: Deron Eriksson <[email protected]> Authored: Thu May 19 15:12:55 2016 -0700 Committer: Deron Eriksson <[email protected]> Committed: Thu May 19 15:12:55 2016 -0700 ---------------------------------------------------------------------- pom.xml | 27 +++++--- src/assembly/distrib.xml | 68 +++--------------- src/assembly/standalone.xml | 72 ++++---------------- .../sysml/api/ml/LogisticRegression.scala | 5 +- .../org/apache/sysml/api/ml/ScriptsUtils.scala | 15 ++-- 5 files changed, 51 insertions(+), 136 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/71733fd1/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b66fd01..8f7bc4e 100644 --- a/pom.xml +++ b/pom.xml @@ -73,15 +73,24 @@ </properties> <build> - - <resources> - <resource> - <directory>scripts/algorithms</directory> - <includes> - <include>**/*.dml</include> - </includes> - </resource> - </resources> + + <!-- Adds scripts to main jar, in-memory jar, sources jar, and standalone jar --> + <resources> + <resource> + <directory>scripts</directory> + <excludes> + <exclude>algorithms/obsolete/*</exclude> + <exclude>algorithms/obsolete</exclude> + <exclude>perftest/*</exclude> + <exclude>perftest</exclude> + <exclude>staging/**/*</exclude> + <exclude>staging</exclude> + <!-- <exclude>*.sh</exclude> --> <!-- applies to sparkDML.sh --> + </excludes> + <targetPath>scripts</targetPath> + </resource> + </resources> + <plugins> <plugin> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/71733fd1/src/assembly/distrib.xml ---------------------------------------------------------------------- diff --git a/src/assembly/distrib.xml b/src/assembly/distrib.xml index b0d3a3e..67a476a 100644 --- a/src/assembly/distrib.xml +++ b/src/assembly/distrib.xml @@ -42,64 +42,16 @@ <fileSet> <directory>${basedir}/scripts</directory> - <includes> - <include>sparkDML.sh</include> - </includes> - <outputDirectory>.</outputDirectory> - </fileSet> - - <fileSet> - <directory>${basedir}/scripts/algorithms</directory> - <includes> - <include>ALS_predict.dml</include> - <include>ALS_topk_predict.dml</include> - <include>ALS.dml</include> - <include>apply-transform.dml</include> - <include>bivar-stats.dml</include> - <include>Cox-predict.dml</include> - <include>Cox.dml</include> - <include>decision-tree-predict.dml</include> - <include>decision-tree.dml</include> - <include>GLM-predict.dml</include> - <include>GLM.dml</include> - <include>KM.dml</include> - <include>Kmeans-predict.dml</include> - <include>Kmeans.dml</include> - <include>l2-svm-predict.dml</include> - <include>l2-svm.dml</include> - <include>LinearRegCG.dml</include> - <include>LinearRegDS.dml</include> - <include>m-svm-predict.dml</include> - <include>m-svm.dml</include> - <include>MultiLogReg.dml</include> - <include>naive-bayes-predict.dml</include> - <include>naive-bayes.dml</include> - <include>PCA.dml</include> - <include>random-forest-predict.dml</include> - <include>random-forest.dml</include> - <include>StepGLM.dml</include> - <include>StepLinearRegDS.dml</include> - <include>stratstats.dml</include> - <include>transform.dml</include> - <include>Univar-Stats.dml</include> - </includes> - <outputDirectory>./algorithms</outputDirectory> - </fileSet> - - <fileSet> - <directory>${basedir}/scripts/utils</directory> - <includes> - <include>cbind.dml</include> - <include>csv2bin.dml</include> - <include>head.dml</include> - <include>project.dml</include> - <include>rowIndexMax.dml</include> - <include>sample.dml</include> - <include>splitXY-dummy.dml</include> - <include>splitXY.dml</include> - <include>write.dml</include> - </includes> - <outputDirectory>./algorithms/utils</outputDirectory> + <excludes> + <exclude>algorithms/obsolete/*</exclude> + <exclude>algorithms/obsolete</exclude> + <exclude>perftest/*</exclude> + <exclude>perftest</exclude> + <exclude>staging/**/*</exclude> + <exclude>staging</exclude> + <!-- <exclude>*.sh</exclude> --> <!-- applies to sparkDML.sh --> + </excludes> + <outputDirectory>scripts</outputDirectory> </fileSet> <fileSet> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/71733fd1/src/assembly/standalone.xml ---------------------------------------------------------------------- diff --git a/src/assembly/standalone.xml b/src/assembly/standalone.xml index 01463a7..8c88cec 100644 --- a/src/assembly/standalone.xml +++ b/src/assembly/standalone.xml @@ -20,7 +20,7 @@ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> - <!-- Assembly file for the "standalone" SystemML release for running on a standalone machine. --> + <!-- Assembly file for the "standalone" tar.gz and zip SystemML releases for running on a standalone machine. --> <id>standalone</id> <formats> @@ -33,65 +33,17 @@ <fileSets> <fileSet> - <directory>${basedir}/scripts/algorithms</directory> - <includes> - <include>ALS_predict.dml</include> - <include>ALS_topk_predict.dml</include> - <include>ALS.dml</include> - <include>apply-transform.dml</include> - <include>bivar-stats.dml</include> - <include>Cox-predict.dml</include> - <include>Cox.dml</include> - <include>decision-tree-predict.dml</include> - <include>decision-tree.dml</include> - <include>GLM-predict.dml</include> - <include>GLM.dml</include> - <include>KM.dml</include> - <include>Kmeans-predict.dml</include> - <include>Kmeans.dml</include> - <include>l2-svm-predict.dml</include> - <include>l2-svm.dml</include> - <include>LinearRegCG.dml</include> - <include>LinearRegDS.dml</include> - <include>m-svm-predict.dml</include> - <include>m-svm.dml</include> - <include>MultiLogReg.dml</include> - <include>naive-bayes-predict.dml</include> - <include>naive-bayes.dml</include> - <include>PCA.dml</include> - <include>random-forest-predict.dml</include> - <include>random-forest.dml</include> - <include>StepGLM.dml</include> - <include>StepLinearRegDS.dml</include> - <include>stratstats.dml</include> - <include>transform.dml</include> - <include>Univar-Stats.dml</include> - </includes> - <outputDirectory>./scripts/algorithms</outputDirectory> - </fileSet> - - <fileSet> - <directory>${basedir}/scripts/datagen</directory> - <includes> - <include>genLinearRegressionData.dml</include> - </includes> - <outputDirectory>./scripts/datagen</outputDirectory> - </fileSet> - - <fileSet> - <directory>${basedir}/scripts/utils</directory> - <includes> - <include>cbind.dml</include> - <include>csv2bin.dml</include> - <include>head.dml</include> - <include>project.dml</include> - <include>rowIndexMax.dml</include> - <include>sample.dml</include> - <include>splitXY-dummy.dml</include> - <include>splitXY.dml</include> - <include>write.dml</include> - </includes> - <outputDirectory>./scripts/utils</outputDirectory> + <directory>${basedir}/scripts</directory> + <excludes> + <exclude>algorithms/obsolete/*</exclude> + <exclude>algorithms/obsolete</exclude> + <exclude>perftest/*</exclude> + <exclude>perftest</exclude> + <exclude>staging/**/*</exclude> + <exclude>staging</exclude> + <!-- <exclude>*.sh</exclude> --> <!-- applies to sparkDML.sh --> + </excludes> + <outputDirectory>scripts</outputDirectory> </fileSet> <fileSet> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/71733fd1/src/main/scala/org/apache/sysml/api/ml/LogisticRegression.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/org/apache/sysml/api/ml/LogisticRegression.scala b/src/main/scala/org/apache/sysml/api/ml/LogisticRegression.scala index 2583088..2fabde1 100644 --- a/src/main/scala/org/apache/sysml/api/ml/LogisticRegression.scala +++ b/src/main/scala/org/apache/sysml/api/ml/LogisticRegression.scala @@ -19,6 +19,7 @@ package org.apache.sysml.api.ml +import java.io.File import org.apache.sysml.api.{ MLContext, MLOutput } import org.apache.sysml.runtime.matrix.MatrixCharacteristics import org.apache.sysml.runtime.instructions.spark.utils.{ RDDConverterUtilsExt => RDDConverterUtils } @@ -60,7 +61,7 @@ trait HasRegParam extends Params { final def getRegParam: Double = $(regParam) } object LogisticRegression { - final val scriptPath = "MultiLogReg.dml" + final val scriptPath = "scripts" + File.separator + "algorithms" + File.separator + "MultiLogReg.dml" } /** @@ -107,7 +108,7 @@ class LogisticRegression(override val uid: String, val sc: SparkContext) extends } } object LogisticRegressionModel { - final val scriptPath = "GLM-predict.dml" + final val scriptPath = "scripts" + File.separator + "algorithms" + File.separator + "GLM-predict.dml" } /** http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/71733fd1/src/main/scala/org/apache/sysml/api/ml/ScriptsUtils.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/org/apache/sysml/api/ml/ScriptsUtils.scala b/src/main/scala/org/apache/sysml/api/ml/ScriptsUtils.scala index 0512287..fdf682d 100644 --- a/src/main/scala/org/apache/sysml/api/ml/ScriptsUtils.scala +++ b/src/main/scala/org/apache/sysml/api/ml/ScriptsUtils.scala @@ -37,23 +37,24 @@ object ScriptsUtils { /* * Internal function to get dml path */ - private[sysml] def resolvePath(filename: String): String = { + private[sysml] def resolvePath(scriptPath: String): String = { import java.io.File - ScriptsUtils.systemmlHome + File.separator + "algorithms" + File.separator + filename + ScriptsUtils.systemmlHome + File.separator + scriptPath } - /* + /* * Internal function to get dml string from jar */ - private[sysml] def getDMLScript(algorithmFileName: String): String = { + private[sysml] def getDMLScript(scriptPath: String): String = { var reader: BufferedReader = null val out = new StringBuilder() try { val in = { if (systemmlHome == null || systemmlHome.equals("")) { - classOf[LogisticRegression].getClassLoader().getResourceAsStream(algorithmFileName) + val resourcePath = "/" + scriptPath.replace("\\", "/") + classOf[LogisticRegression].getResourceAsStream(resourcePath) } else { - new java.io.FileInputStream(resolvePath(algorithmFileName)) + new java.io.FileInputStream(resolvePath(scriptPath)) } } var reader = new BufferedReader(new InputStreamReader(in)) @@ -65,7 +66,7 @@ object ScriptsUtils { } } catch { case ex: Exception => - throw new DMLRuntimeException("Cannot read the algorithm file " + algorithmFileName, ex) + throw new DMLRuntimeException("Cannot read the script file " + scriptPath, ex) } finally { if (reader != null) reader.close();
