Repository: systemml Updated Branches: refs/heads/master 820b53046 -> ae819a253
[Minor] update creation of lite jar Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/ae819a25 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/ae819a25 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/ae819a25 Branch: refs/heads/master Commit: ae819a253097da796e689764e4926d46228e055e Parents: 820b530 Author: Berthold Reinwald <[email protected]> Authored: Fri Nov 3 22:24:04 2017 -0700 Committer: Berthold Reinwald <[email protected]> Committed: Fri Nov 3 22:30:59 2017 -0700 ---------------------------------------------------------------------- pom.xml | 308 ++++++++++--------- src/assembly/lite.xml | 22 +- .../org/apache/sysml/utils/lite/BuildLite.java | 22 +- 3 files changed, 198 insertions(+), 154 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/ae819a25/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ff90986..6201c77 100644 --- a/pom.xml +++ b/pom.xml @@ -1,23 +1,17 @@ <?xml version="1.0" encoding="UTF-8"?> -<!-- - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<!-- * Licensed to the Apache Software Foundation (ASF) under one * or more + contributor license agreements. See the NOTICE file * distributed with this + work for additional information * regarding copyright ownership. The ASF + licenses this file * to you under the Apache License, Version 2.0 (the * + "License"); you may not use this file except in compliance * with the License. + You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, * software + distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the + License for the * specific language governing permissions and limitations + * under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache</groupId> @@ -78,45 +72,46 @@ </properties> <repositories> - <repository> - <id>central</id> - <url>https://repo1.maven.org/maven2</url> - <releases> - <enabled>true</enabled> - </releases> - </repository> - </repositories> + <repository> + <id>central</id> + <url>https://repo1.maven.org/maven2</url> + <releases> + <enabled>true</enabled> + </releases> + </repository> + </repositories> <build> - <!-- Adds scripts to main jar, in-memory jar, sources jar, and standalone jar --> - <resources> - <resource> - <directory>scripts</directory> - <excludes> - <exclude>algorithms/obsolete/*</exclude> - <exclude>datagen/obsolete/*</exclude> - <exclude>perftest/*</exclude> - <exclude>staging/**/*</exclude> - <exclude>nn/test/compare_backends/*</exclude> - <exclude>nn/test/compare_backends/*</exclude> - <exclude>nn/examples/caffe2dml/**/*</exclude> - <!-- <exclude>*.sh</exclude> --> <!-- applies to sparkDML.sh --> - </excludes> - <targetPath>scripts</targetPath> - </resource> - <resource> - <directory>src/main/cpp/kernels</directory> - <excludes> - <exclude>*.cu</exclude> - </excludes> - <targetPath>kernels</targetPath> - </resource> - <resource> - <directory>src/main/cpp/lib</directory> - <targetPath>lib</targetPath> - </resource> - </resources> + <!-- Adds scripts to main jar, in-memory jar, sources jar, and standalone + jar --> + <resources> + <resource> + <directory>scripts</directory> + <excludes> + <exclude>algorithms/obsolete/*</exclude> + <exclude>datagen/obsolete/*</exclude> + <exclude>perftest/*</exclude> + <exclude>staging/**/*</exclude> + <exclude>nn/test/compare_backends/*</exclude> + <exclude>nn/test/compare_backends/*</exclude> + <exclude>nn/examples/caffe2dml/**/*</exclude> + <!-- <exclude>*.sh</exclude> --> <!-- applies to sparkDML.sh --> + </excludes> + <targetPath>scripts</targetPath> + </resource> + <resource> + <directory>src/main/cpp/kernels</directory> + <excludes> + <exclude>*.cu</exclude> + </excludes> + <targetPath>kernels</targetPath> + </resource> + <resource> + <directory>src/main/cpp/lib</directory> + <targetPath>lib</targetPath> + </resource> + </resources> <plugins> <plugin> @@ -180,8 +175,8 @@ </executions> <configuration> - <!-- Include signature files so that recent versions of Java will run - the resulting jar without complaining about "Invalid signature file digest + <!-- Include signature files so that recent versions of Java will run + the resulting jar without complaining about "Invalid signature file digest for Manifest main attributes". --> <filters> <filter> @@ -209,8 +204,8 @@ </goals> <configuration> <sources> - <!-- Test suite classes kept in a separate folder for compatibility - with the Eclipse JUnit test runner. --> + <!-- Test suite classes kept in a separate folder for compatibility + with the Eclipse JUnit test runner. --> <source>${basedir}/src/test_suites/java</source> </sources> </configuration> @@ -226,55 +221,57 @@ </configuration> </plugin> <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <version>3.2.2</version> - <executions> - <execution> - <id>eclipse-add-source</id> - <goals> - <goal>add-source</goal> - </goals> - </execution> - <execution> - <id>scala-compile-first</id> - <phase>process-resources</phase> - <goals> - <goal>compile</goal> - </goals> - </execution> - <execution> - <id>scala-test-compile-first</id> - <phase>process-test-resources</phase> - <goals> - <goal>testCompile</goal> - </goals> - </execution> - </executions> + <groupId>net.alchim31.maven</groupId> + <artifactId>scala-maven-plugin</artifactId> + <version>3.2.2</version> + <executions> + <execution> + <id>eclipse-add-source</id> + <goals> + <goal>add-source</goal> + </goals> + </execution> + <execution> + <id>scala-compile-first</id> + <phase>process-resources</phase> + <goals> + <goal>compile</goal> + </goals> + </execution> + <execution> + <id>scala-test-compile-first</id> + <phase>process-test-resources</phase> + <goals> + <goal>testCompile</goal> + </goals> + </execution> + </executions> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> <executions> - <execution> - <id>default-jar</id> - <goals><goal>jar</goal></goals> - <phase>package</phase> - <configuration> - <archive> - <index>true</index> - <manifestEntries> - <Build-Timestamp>${maven.build.timestamp}</Build-Timestamp> - <Main-Class>org.apache.sysml.api.DMLScript</Main-Class> - </manifestEntries> - </archive> - <excludes> - <exclude>**/caffe/*</exclude> - <exclude>**/org/apache/sysml/api/dl/*</exclude> - </excludes> - </configuration> - </execution> - </executions> + <execution> + <id>default-jar</id> + <goals> + <goal>jar</goal> + </goals> + <phase>package</phase> + <configuration> + <archive> + <index>true</index> + <manifestEntries> + <Build-Timestamp>${maven.build.timestamp}</Build-Timestamp> + <Main-Class>org.apache.sysml.api.DMLScript</Main-Class> + </manifestEntries> + </archive> + <excludes> + <exclude>**/caffe/*</exclude> + <exclude>**/org/apache/sysml/api/dl/*</exclude> + </excludes> + </configuration> + </execution> + </executions> </plugin> <plugin> @@ -343,50 +340,51 @@ </plugin> <plugin> - <groupId>com.github.os72</groupId> - <artifactId>protoc-jar-maven-plugin</artifactId> - <version>3.4.0.1</version> - <executions> - <execution> - <id>caffe-sources</id> - <phase>generate-sources</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <protocVersion>2.5.0</protocVersion> <!-- 2.4.1, 2.5.0, 2.6.1, 3.4.0 --> - <inputDirectories> - <include>src/main/proto/caffe</include> - </inputDirectories> - <outputDirectories> - <include>src/main/java</include> - </outputDirectories> - </configuration> - </execution> - </executions> + <groupId>com.github.os72</groupId> + <artifactId>protoc-jar-maven-plugin</artifactId> + <version>3.4.0.1</version> + <executions> + <execution> + <id>caffe-sources</id> + <phase>generate-sources</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <protocVersion>2.5.0</protocVersion> <!-- 2.4.1, 2.5.0, 2.6.1, 3.4.0 --> + <inputDirectories> + <include>src/main/proto/caffe</include> + </inputDirectories> + <outputDirectories> + <include>src/main/java</include> + </outputDirectories> + </configuration> + </execution> + </executions> </plugin> <!-- Currently, all tests are integration tests. --> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>2.18</version><!--$NO-MVN-MAN-VER$--> + <version>2.18</version><!--$NO-MVN-MAN-VER$ --> <configuration> <!-- STDERR/STDOUT to individual .txt files instead of console --> <redirectTestOutputToFile>true</redirectTestOutputToFile> - <!-- Run (num cores) test harness classes in parallel, each in a separate + <!-- Run (num cores) test harness classes in parallel, each in a separate process --> <forkCount>1C</forkCount> <reuseForks>true</reuseForks> - <!-- Third argument prevents Java from popping up lots of windows on + <!-- Third argument prevents Java from popping up lots of windows on MacOS --> - <argLine>-Dfile.encoding=UTF-8 -Xmx2g -Xms2g -Xmn200m -Djava.awt.headless=true</argLine> + <argLine>-Dfile.encoding=UTF-8 -Xmx2g -Xms2g -Xmn200m + -Djava.awt.headless=true</argLine> <includes> - <!-- All tests are integration tests as far as Maven is concerned. --> + <!-- All tests are integration tests as far as Maven is concerned. --> </includes> <excludes> @@ -405,7 +403,7 @@ <!-- *** HACK ALERT *** HACK ALERT *** HACK ALERT *** --> <!-- Failsafe 2.18 has a bug in computing # cores, so use 2.17 --> - <version>2.17</version><!--$NO-MVN-MAN-VER$--> + <version>2.17</version><!--$NO-MVN-MAN-VER$ --> <!-- *** END HACK *** --> <executions> @@ -429,7 +427,7 @@ <forkCount>0.5C</forkCount> <reuseForks>false</reuseForks> - <!-- Last argument prevents Java from popping up lots of windows on + <!-- Last argument prevents Java from popping up lots of windows on MacOS --> <argLine>-Dfile.encoding=UTF-8 -Xmx2g -Xms2g -Xmn200m ${integrationTestExtraJVMArgs} @@ -479,8 +477,8 @@ </configuration> </execution> <execution> - <!-- NOTE: We don't clean up systemml.egg-info since this makes it difficult - to uninstall a dev version after a maven clean --> + <!-- NOTE: We don't clean up systemml.egg-info since this makes it + difficult to uninstall a dev version after a maven clean --> <id>clean-python-files</id> <phase>clean</phase> <goals> @@ -582,7 +580,9 @@ <phase>package</phase> <configuration> <target name="copy and rename JAR"> - <copy file="${project.build.directory}/${project.artifactId}-${project.version}.jar" tofile="${project.build.directory}/SystemML.jar" /> + <copy + file="${project.build.directory}/${project.artifactId}-${project.version}.jar" + tofile="${project.build.directory}/SystemML.jar" /> </target> </configuration> <goals> @@ -702,7 +702,7 @@ </profile> <profile> - <!-- Profile for Windows builds. Not currently needed, but might be needed + <!-- Profile for Windows builds. Not currently needed, but might be needed in the future. --> <id>platform-windows</id> <activation> @@ -711,7 +711,7 @@ </os> </activation> <properties> - <!-- Extra JVM args for Windows go here.--> + <!-- Extra JVM args for Windows go here. --> <integrationTestExtraJVMArgs /> </properties> </profile> @@ -852,7 +852,7 @@ <exclude>**/*.keep</exclude> <exclude>**/target/**</exclude> <exclude>**/README.md</exclude> - <exclude>**/*.svg</exclude> + <exclude>**/*.svg</exclude> <!-- Jupyter Notebooks --> <exclude>**/*.ipynb</exclude> <!-- Generated antlr files --> @@ -898,17 +898,15 @@ <profile> <!-- Can be used to ignore doclint javadoc issues --> <id>ignore-doclint</id> - <!-- <activation> - <jdk>[1.8,)</jdk> - </activation> --> + <!-- <activation> <jdk>[1.8,)</jdk> </activation> --> <properties> <javadoc.opts>-Xdoclint:none</javadoc.opts> </properties> </profile> <profile> - <!-- Profile to create binary distributions. - Execute with `mvn clean package -P distribution` --> + <!-- Profile to create binary distributions. Execute with `mvn clean package + -P distribution` --> <id>distribution</id> <build> <plugins> @@ -1017,13 +1015,10 @@ <artifactId>maven-javadoc-plugin</artifactId> <version>2.10.3</version> <configuration> - <!-- Need to include the following packages, so exclude others: - org.apache.sysml.api - org.apache.sysml.runtime.instructions.spark.utils (for RDDConverterUtils, etc) - org.apache.sysml.runtime.matrix (for MatrixCharacteristics, etc) - org.apache.sysml.runtime.matrix.data (for MatrixIndexes, MatrixBlock, etc) - org.apache.sysml.udf - --> + <!-- Need to include the following packages, so exclude others: org.apache.sysml.api + org.apache.sysml.runtime.instructions.spark.utils (for RDDConverterUtils, + etc) org.apache.sysml.runtime.matrix (for MatrixCharacteristics, etc) org.apache.sysml.runtime.matrix.data + (for MatrixIndexes, MatrixBlock, etc) org.apache.sysml.udf --> <excludePackageNames>caffe:org.apache.sysml.conf:org.apache.sysml.debug:org.apache.sysml.hops:org.apache.sysml.lops:org.apache.sysml.parser:org.apache.sysml.runtime.controlprogram:org.apache.sysml.runtime.functionobjects:org.apache.sysml.runtime.instructions.cp:org.apache.sysml.runtime.instructions.cpfile:org.apache.sysml.runtime.instructions.mr:org.apache.sysml.runtime.instructions.spark.data:org.apache.sysml.runtime.instructions.spark.functions:org.apache.sysml.runtime.io:org.apache.sysml.runtime.matrix.data.hadoopfix:org.apache.sysml.runtime.matrix.mapred:org.apache.sysml.runtime.matrix.operators:org.apache.sysml.runtime.matrix.sort:org.apache.sysml.runtime.transform:org.apache.sysml.runtime.util:org.apache.sysml.utils:org.apache.sysml.yarn</excludePackageNames> <additionalparam>${javadoc.opts}</additionalparam> </configuration> @@ -1162,6 +1157,17 @@ <descriptors> <descriptor>src/assembly/lite.xml</descriptor> </descriptors> + <archive> + <manifest> + <mainClass>org.apache.sysml.api.DMLScript</mainClass> + </manifest> + <manifestEntries> + <Build-Time>${maven.build.timestamp}</Build-Time> + <Group-Id>${project.groupId}</Group-Id> + <Artifact-Id>${project.artifactId}-lite</Artifact-Id> + <Version>${project.version}</Version> + </manifestEntries> + </archive> </configuration> </execution> </executions> @@ -1171,8 +1177,8 @@ </profile> <profile> - <!-- Profile to create standalone jar. - Execute with `mvn clean package -P standalone-jar` --> + <!-- Profile to create standalone jar. Execute with `mvn clean package + -P standalone-jar` --> <id>standalone-jar</id> <build> <plugins> http://git-wip-us.apache.org/repos/asf/systemml/blob/ae819a25/src/assembly/lite.xml ---------------------------------------------------------------------- diff --git a/src/assembly/lite.xml b/src/assembly/lite.xml index 908c72e..0212ace 100644 --- a/src/assembly/lite.xml +++ b/src/assembly/lite.xml @@ -53,7 +53,7 @@ <dependencySets> <dependencySet> <includes> - <include>*:antlr4-runtime</include> + <include>*:antlr4</include> </includes> <unpackOptions> <includes> @@ -210,6 +210,8 @@ </includes> <unpackOptions> <includes> + <include>org/apache/commons/cli/OptionValidator.class</include> + <include>org/apache/commons/cli/Util.class</include> <include>org/apache/commons/cli/AlreadySelectedException.class</include> <include>org/apache/commons/cli/CommandLine.class</include> <include>org/apache/commons/cli/CommandLineParser.class</include> @@ -620,7 +622,12 @@ <unpackOptions> <includes> <include>META-INF/services/org.apache.hadoop.fs.FileSystem</include> + <include>common-version-info.properties</include> <include>org/apache/hadoop/log/metrics/EventCounter.class</include> + <include>org/apache/hadoop/security/Groups$CachedGroups.class</include> + <include>org/apache/hadoop/security/token/Token.class</include> + <include>org/apache/hadoop/security/token/TokenIdentifier.class</include> + <include>org/apache/hadoop/util/GenericOptionsParser.class</include> <include>org/apache/hadoop/util/ShutdownHookManager$2.class</include> <include>org/apache/hadoop/HadoopIllegalArgumentException.class</include> <include>org/apache/hadoop/conf/Configurable.class</include> @@ -1027,6 +1034,19 @@ <dependencySet> <includes> + <include>*:hadoop-mapreduce-client-jobclient</include> + </includes> + <unpackOptions> + <includes> + <include>org/apache/hadoop/mapred/YarnClientProtocolProvider.class</include> + </includes> + </unpackOptions> + <scope>compile</scope> + <unpack>true</unpack> + </dependencySet> + + <dependencySet> + <includes> <include>*:jcl-over-slf4j</include> </includes> <unpackOptions> http://git-wip-us.apache.org/repos/asf/systemml/blob/ae819a25/src/main/java/org/apache/sysml/utils/lite/BuildLite.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/lite/BuildLite.java b/src/main/java/org/apache/sysml/utils/lite/BuildLite.java index 9048ce6..ad76085 100644 --- a/src/main/java/org/apache/sysml/utils/lite/BuildLite.java +++ b/src/main/java/org/apache/sysml/utils/lite/BuildLite.java @@ -92,6 +92,14 @@ public class BuildLite { additionalResources.add("org/apache/hadoop/util/ShutdownHookManager$2.class"); additionalResources.add("org/apache/hadoop/log/metrics/EventCounter.class"); + additionalResources.add("org/apache/hadoop/util/GenericOptionsParser.class"); + additionalResources.add("org/apache/hadoop/security/token/Token.class"); + additionalResources.add("org/apache/hadoop/security/token/TokenIdentifier.class"); + additionalResources.add("org/apache/hadoop/security/Groups$CachedGroups.class"); + additionalResources.add("org/apache/commons/cli/OptionValidator.class"); + additionalResources.add("org/apache/commons/cli/Util.class"); + additionalResources.add("common-version-info.properties"); + } /** @@ -103,8 +111,18 @@ public class BuildLite { SortedSet<String> hadoopCommonResources = new TreeSet<>(); hadoopCommonResources.add("META-INF/services/org.apache.hadoop.fs.FileSystem"); hadoopCommonResources.add("org/apache/hadoop/util/ShutdownHookManager$2.class"); - hadoopCommonResources.add("org/apache/hadoop/log/metrics/EventCounter.class"); - additionalJarToFileMappingsForDependencySets.put("hadoop-common", hadoopCommonResources); + hadoopCommonResources.add("org/apache/hadoop/log/metrics/EventCounter.class"); + hadoopCommonResources.add("org/apache/hadoop/util/GenericOptionsParser.class"); + hadoopCommonResources.add("org/apache/hadoop/security/token/Token.class"); + hadoopCommonResources.add("org/apache/hadoop/security/token/TokenIdentifier.class"); + hadoopCommonResources.add("org/apache/hadoop/security/Groups$CachedGroups.class"); + hadoopCommonResources.add("common-version-info.properties"); + additionalJarToFileMappingsForDependencySets.put("hadoop-common", hadoopCommonResources); + + SortedSet<String> commonsCliResources = new TreeSet<String>(); + commonsCliResources.add("org/apache/commons/cli/OptionValidator.class"); + commonsCliResources.add("org/apache/commons/cli/Util.class"); + additionalJarToFileMappingsForDependencySets.put("commons-cli", commonsCliResources); } /**
