Repository: spark Updated Branches: refs/heads/master e26dac5fe -> d3f4a2119
[SPARK-15526][ML][FOLLOWUP] Make JPMML provided scope to avoid including unshaded JARs, and repromote to compile in MLlib Following the comment at https://issues.apache.org/jira/browse/SPARK-15526?focusedCommentId=16086106&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-16086106 -- this change actually needed a little more work to be complete. This also marks JPMML as `provided` to make sure its JARs aren't included in the `jars` output, but then scopes to `compile` in `mllib`. This is how Guava is handled. Checked result in `assembly/target/scala-2.11/jars` to verify there are no JPMML jars. Maven and SBT builds still work. Author: Sean Owen <[email protected]> Closes #18637 from srowen/SPARK-15526.2. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d3f4a211 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d3f4a211 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d3f4a211 Branch: refs/heads/master Commit: d3f4a2119634134320ebeb1ddbe0c49db080c040 Parents: e26dac5 Author: Sean Owen <[email protected]> Authored: Tue Jul 18 09:53:49 2017 -0700 Committer: Marcelo Vanzin <[email protected]> Committed: Tue Jul 18 09:53:51 2017 -0700 ---------------------------------------------------------------------- core/pom.xml | 1 + dev/deps/spark-deps-hadoop-2.6 | 2 -- dev/deps/spark-deps-hadoop-2.7 | 2 -- .../spark/launcher/AbstractCommandBuilder.java | 1 + mllib/pom.xml | 38 ++++++++++++++++---- pom.xml | 21 +++++++++-- 6 files changed, 52 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 91ee941..bc6b1c4 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -429,6 +429,7 @@ <!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use shaded deps. So here we store jars in their original form which are added when the classpath is computed. --> + <!-- See similar execution in mllib/pom.xml --> <execution> <id>copy-dependencies</id> <phase>package</phase> http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/dev/deps/spark-deps-hadoop-2.6 ---------------------------------------------------------------------- diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 1a6515b..76e3764 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -158,8 +158,6 @@ parquet-format-2.3.1.jar parquet-hadoop-1.8.2.jar parquet-hadoop-bundle-1.6.0.jar parquet-jackson-1.8.2.jar -pmml-model-1.2.15.jar -pmml-schema-1.2.15.jar protobuf-java-2.5.0.jar py4j-0.10.6.jar pyrolite-4.13.jar http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/dev/deps/spark-deps-hadoop-2.7 ---------------------------------------------------------------------- diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index 09e5a42..f417347 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -159,8 +159,6 @@ parquet-format-2.3.1.jar parquet-hadoop-1.8.2.jar parquet-hadoop-bundle-1.6.0.jar parquet-jackson-1.8.2.jar -pmml-model-1.2.15.jar -pmml-schema-1.2.15.jar protobuf-java-2.5.0.jar py4j-0.10.6.jar pyrolite-4.13.jar http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java ---------------------------------------------------------------------- diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index 481ff20..8004247 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -185,6 +185,7 @@ abstract class AbstractCommandBuilder { // Add this path to include jars that are shaded in the final deliverable created during // the maven build. These jars are copied to this directory during the build. addToClassPath(cp, String.format("%s/core/target/jars/*", sparkHome)); + addToClassPath(cp, String.format("%s/mllib/target/jars/*", sparkHome)); } // Add Spark jars to the classpath. For the testing case, we rely on the test code to set and http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/mllib/pom.xml ---------------------------------------------------------------------- diff --git a/mllib/pom.xml b/mllib/pom.xml index 572670d..c72a16a 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -101,13 +101,7 @@ <dependency> <groupId>org.jpmml</groupId> <artifactId>pmml-model</artifactId> - <version>1.2.15</version> - <exclusions> - <exclusion> - <groupId>org.jpmml</groupId> - <artifactId>pmml-agent</artifactId> - </exclusion> - </exclusions> + <scope>compile</scope> </dependency> <dependency> <groupId>org.apache.spark</groupId> @@ -139,8 +133,38 @@ </dependencies> </profile> </profiles> + <build> <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use + shaded deps. So here we store jars in their original form which are added + when the classpath is computed. --> + <!-- See similar execution in core/pom.xml --> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <outputDirectory>${project.build.directory}</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + <useSubDirectoryPerType>true</useSubDirectoryPerType> + <includeGroupIds>org.jpmml</includeGroupIds> + <silent>true</silent> + </configuration> + </execution> + </executions> + </plugin> + </plugins> </build> + </project> http://git-wip-us.apache.org/repos/asf/spark/blob/d3f4a211/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 11cd028..1b81263 100644 --- a/pom.xml +++ b/pom.xml @@ -382,6 +382,18 @@ <version>14.0.1</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.jpmml</groupId> + <artifactId>pmml-model</artifactId> + <version>1.2.15</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>org.jpmml</groupId> + <artifactId>pmml-agent</artifactId> + </exclusion> + </exclusions> + </dependency> <!-- End of shaded deps --> <dependency> <groupId>org.apache.commons</groupId> @@ -2511,8 +2523,8 @@ <profiles> <!-- - This profile is enabled automatically by the sbt built. It changes the scope for the guava - dependency, since we don't shade it in the artifacts generated by the sbt build. + This profile is enabled automatically by the sbt build. It changes the scope for shaded + dependencies, since we don't shade it in the artifacts generated by the sbt build. --> <profile> <id>sbt</id> @@ -2522,6 +2534,11 @@ <artifactId>guava</artifactId> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.jpmml</groupId> + <artifactId>pmml-model</artifactId> + <scope>compile</scope> + </dependency> </dependencies> </profile> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
