Repository: incubator-systemml Updated Branches: refs/heads/master 39a37ae40 -> 738ddfc17
[SYSTEMML-692] Separated Caffe2dml into a separate jar 1. Caffe2DML should be integrated in main SystemML jar. Pros: One jar for all. Cons: +1 MB. 2. Caffe2DML should be released as a separate jar. Pros: -1MB. Cons: One more artifact to release. 3 Caffe2DML should be a separate project. I don't like this option as Caffe2DML is an API of SystemML, not a standalone tool. This commit uses option 2. We can later revisit this if other committers have strong opinion. Closes #466. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/738ddfc1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/738ddfc1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/738ddfc1 Branch: refs/heads/master Commit: 738ddfc17ca56d7c366bbea83f584e9f10cd8d44 Parents: 39a37ae Author: Niketan Pansare <[email protected]> Authored: Sun Apr 30 12:27:26 2017 -0700 Committer: Niketan Pansare <[email protected]> Committed: Sun Apr 30 12:27:26 2017 -0700 ---------------------------------------------------------------------- pom.xml | 45 ++++++++++++++-- src/main/python/pre_setup.py | 3 ++ src/main/python/systemml/classloader.py | 57 ++++++++++++-------- .../apache/sysml/api/dl/Caffe2DMLLoader.scala | 37 +++++++++++++ 4 files changed, 117 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/738ddfc1/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 9a33853..af788e1 100644 --- a/pom.xml +++ b/pom.xml @@ -254,15 +254,51 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> - <configuration> - <archive> + <executions> + <execution> + <goals><goal>jar</goal></goals> + <phase>package</phase> + <configuration> + <archive> <index>true</index> <manifestEntries> <Build-Timestamp>${maven.build.timestamp}</Build-Timestamp> <Main-Class>org.apache.sysml.api.DMLScript</Main-Class> </manifestEntries> - </archive> - </configuration> + </archive> + <classifier>lite</classifier> + <excludes> + <exclude>**/caffe/*</exclude> + <exclude>**/org/tensorflow/*</exclude> + <exclude>**/org/tensorflow/framework/*</exclude> + <exclude>**/org/tensorflow/util/*</exclude> + <exclude>**/org/apache/sysml/api/dl/*</exclude> + </excludes> + </configuration> + </execution> + <execution> + <id>extra</id> + <goals><goal>jar</goal></goals> + <phase>package</phase> + <configuration> + <archive> + <index>true</index> + <manifestEntries> + <Build-Timestamp>${maven.build.timestamp}</Build-Timestamp> + <Main-Class>org.apache.sysml.api.DMLScript</Main-Class> + </manifestEntries> + </archive> + <classifier>extra</classifier> + <includes> + <include>**/META-INF/*</include> + <include>**/caffe/*</include> + <include>**/org/tensorflow/framework/*</include> + <include>**/org/tensorflow/util/*</include> + <include>**/org/apache/sysml/api/dl/*</include> + </includes> + </configuration> + </execution> + </executions> </plugin> <plugin> @@ -582,6 +618,7 @@ <phase>package</phase> <configuration> <target name="copy and rename JAR"> + <move file="${project.build.directory}/${project.artifactId}-${project.version}-lite.jar" tofile="${project.build.directory}/${project.artifactId}-${project.version}.jar" /> <copy file="${project.build.directory}/${project.artifactId}-${project.version}.jar" tofile="${project.build.directory}/SystemML.jar" /> </target> </configuration> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/738ddfc1/src/main/python/pre_setup.py ---------------------------------------------------------------------- diff --git a/src/main/python/pre_setup.py b/src/main/python/pre_setup.py index 5443a6d..cb421a9 100644 --- a/src/main/python/pre_setup.py +++ b/src/main/python/pre_setup.py @@ -33,3 +33,6 @@ for file in os.listdir(os.path.join(root_dir, 'target')): if fnmatch.fnmatch(file, 'systemml-*-incubating-SNAPSHOT.jar') or fnmatch.fnmatch(file, 'systemml-*-incubating.jar'): shutil.copyfile(os.path.join(root_dir, 'target', file), os.path.join(java_dir_full_path, file)) + if fnmatch.fnmatch(file, 'systemml-*-incubating-SNAPSHOT-extra.jar') or fnmatch.fnmatch(file, 'systemml-*-incubating-extra.jar'): + shutil.copyfile(os.path.join(root_dir, 'target', file), + os.path.join(java_dir_full_path, file)) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/738ddfc1/src/main/python/systemml/classloader.py ---------------------------------------------------------------------- diff --git a/src/main/python/systemml/classloader.py b/src/main/python/systemml/classloader.py index 79c0aad..45fe4de 100644 --- a/src/main/python/systemml/classloader.py +++ b/src/main/python/systemml/classloader.py @@ -38,6 +38,29 @@ def _createJavaObject(sc, obj_type): else: raise ValueError('Incorrect usage: supported values: mlcontext or dummy') +def _getJarFileName(sc, suffix): + import imp, fnmatch + jar_file_name = '_ignore.jar' + java_dir = os.path.join(imp.find_module("systemml")[1], "systemml-java") + for file in os.listdir(java_dir): + if fnmatch.fnmatch(file, 'systemml-*-incubating-SNAPSHOT' + suffix + '.jar') or fnmatch.fnmatch(file, 'systemml-*-incubating' + suffix + '.jar'): + jar_file_name = os.path.join(java_dir, file) + return jar_file_name + +def _getLoaderInstance(sc, jar_file_name, className, hint): + err_msg = 'Unable to load systemml-*.jar into current pyspark session.' + if os.path.isfile(jar_file_name): + sc._jsc.addJar(jar_file_name) + jar_file_url = sc._jvm.java.io.File(jar_file_name).toURI().toURL() + url_class = sc._jvm.java.net.URL + jar_file_url_arr = sc._gateway.new_array(url_class, 1) + jar_file_url_arr[0] = jar_file_url + url_class_loader = sc._jvm.java.net.URLClassLoader(jar_file_url_arr, sc._jsc.getClass().getClassLoader()) + c1 = sc._jvm.java.lang.Class.forName(className, True, url_class_loader) + return c1.newInstance() + else: + raise ImportError(err_msg + ' Hint: Download the jar from http://systemml.apache.org/download and ' + hint ) + def createJavaObject(sc, obj_type): """ Performs appropriate check if SystemML.jar is available and returns the handle to MLContext object on JVM @@ -51,27 +74,19 @@ def createJavaObject(sc, obj_type): try: return _createJavaObject(sc, obj_type) except (py4j.protocol.Py4JError, TypeError): - import imp, fnmatch - jar_file_name = '_ignore.jar' - java_dir = os.path.join(imp.find_module("systemml")[1], "systemml-java") - for file in os.listdir(java_dir): - if fnmatch.fnmatch(file, 'systemml-*-incubating-SNAPSHOT.jar') or fnmatch.fnmatch(file, 'systemml-*-incubating.jar'): - jar_file_name = os.path.join(java_dir, file) - err_msg = 'Unable to load SystemML.jar into current pyspark session.' + ret = None + err_msg = 'Unable to load systemml-*.jar into current pyspark session.' hint = 'Provide the following argument to pyspark: --driver-class-path ' - if os.path.isfile(jar_file_name): - sc._jsc.addJar(jar_file_name) - jar_file_url = sc._jvm.java.io.File(jar_file_name).toURI().toURL() - url_class = sc._jvm.java.net.URL - jar_file_url_arr = sc._gateway.new_array(url_class, 1) - jar_file_url_arr[0] = jar_file_url - url_class_loader = sc._jvm.java.net.URLClassLoader(jar_file_url_arr, sc._jsc.getClass().getClassLoader()) - c1 = sc._jvm.java.lang.Class.forName('org.apache.sysml.utils.SystemMLLoaderUtils', True, url_class_loader) - x = c1.newInstance() - x.loadSystemML(jar_file_name) - else: - raise ImportError(err_msg + ' Hint: Download the jar from http://systemml.apache.org/download and ' + hint + 'SystemML.jar') + # First load SystemML + jar_file_name = _getJarFileName(sc, '') + x = _getLoaderInstance(sc, jar_file_name, 'org.apache.sysml.utils.SystemMLLoaderUtils', hint + 'SystemML.jar') + x.loadSystemML(jar_file_name) try: - return _createJavaObject(sc, obj_type) + ret = _createJavaObject(sc, obj_type) except (py4j.protocol.Py4JError, TypeError): - raise ImportError(err_msg + ' Hint: ' + hint + jar_file_name) \ No newline at end of file + raise ImportError(err_msg + ' Hint: ' + hint + jar_file_name) + # Now load caffe2dml + jar_file_name = _getJarFileName(sc, '-extra') + x = _getLoaderInstance(sc, jar_file_name, 'org.apache.sysml.api.dl.Caffe2DMLLoader', hint + 'systemml-*-extra.jar') + x.loadCaffe2DML(jar_file_name) + return ret \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/738ddfc1/src/main/scala/org/apache/sysml/api/dl/Caffe2DMLLoader.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/org/apache/sysml/api/dl/Caffe2DMLLoader.scala b/src/main/scala/org/apache/sysml/api/dl/Caffe2DMLLoader.scala new file mode 100644 index 0000000..30d86fd --- /dev/null +++ b/src/main/scala/org/apache/sysml/api/dl/Caffe2DMLLoader.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.api.dl + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.io.File; + +class Caffe2DMLLoader { + def loadCaffe2DML(filePath:String):Unit = { + val url = new File(filePath).toURI().toURL(); + val classLoader = ClassLoader.getSystemClassLoader().asInstanceOf[URLClassLoader]; + val method = classOf[URLClassLoader].getDeclaredMethod("addURL", classOf[URL]); + method.setAccessible(true); + method.invoke(classLoader, url); + } +} \ No newline at end of file
