Repository: spark Updated Branches: refs/heads/master e258e5040 -> 856c50f59
SPARK-1387. Update build plugins, avoid plugin version warning, centralize versions Another handful of small build changes to organize and standardize a bit, and avoid warnings: - Update Maven plugin versions for good measure - Since plugins need maven 3.0.4 already, require it explicitly (<3.0.4 had some bugs anyway) - Use variables to define versions across dependencies where they should move in lock step - ... and make this consistent between Maven/SBT OK, I also updated the JIRA URL while I was at it here. Author: Sean Owen <so...@cloudera.com> Closes #291 from srowen/SPARK-1387 and squashes the following commits: 461eca1 [Sean Owen] Couldn't resist also updating JIRA location to new one c2d5cc5 [Sean Owen] Update plugins and Maven version; use variables consistently across Maven/SBT to define dependency versions that should stay in step. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/856c50f5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/856c50f5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/856c50f5 Branch: refs/heads/master Commit: 856c50f59bffbf76ad495eaab837febaf65cf02d Parents: e258e50 Author: Sean Owen <so...@cloudera.com> Authored: Sun Apr 6 17:40:37 2014 -0700 Committer: Patrick Wendell <pwend...@gmail.com> Committed: Sun Apr 6 17:41:01 2014 -0700 ---------------------------------------------------------------------- assembly/pom.xml | 2 +- core/pom.xml | 2 -- dev/audit-release/maven_app_core/pom.xml | 2 +- docs/building-with-maven.md | 2 +- examples/pom.xml | 2 +- graphx/pom.xml | 2 +- mllib/pom.xml | 2 +- pom.xml | 43 ++++++++++++----------- project/SparkBuild.scala | 49 +++++++++++++++------------ streaming/pom.xml | 1 - 10 files changed, 57 insertions(+), 50 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/856c50f5/assembly/pom.xml ---------------------------------------------------------------------- diff --git a/assembly/pom.xml b/assembly/pom.xml index b5e752c..255107a 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -208,7 +208,7 @@ <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>buildnumber-maven-plugin</artifactId> - <version>1.1</version> + <version>1.2</version> <executions> <execution> <phase>validate</phase> http://git-wip-us.apache.org/repos/asf/spark/blob/856c50f5/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 66f9fc4..1f80838 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -117,12 +117,10 @@ <dependency> <groupId>com.twitter</groupId> <artifactId>chill_${scala.binary.version}</artifactId> - <version>0.3.1</version> </dependency> <dependency> <groupId>com.twitter</groupId> <artifactId>chill-java</artifactId> - <version>0.3.1</version> </dependency> <dependency> <groupId>commons-net</groupId> http://git-wip-us.apache.org/repos/asf/spark/blob/856c50f5/dev/audit-release/maven_app_core/pom.xml ---------------------------------------------------------------------- diff --git a/dev/audit-release/maven_app_core/pom.xml b/dev/audit-release/maven_app_core/pom.xml index 0b837c0..76a381f 100644 --- a/dev/audit-release/maven_app_core/pom.xml +++ b/dev/audit-release/maven_app_core/pom.xml @@ -49,7 +49,7 @@ <plugins> <plugin> <artifactId>maven-compiler-plugin</artifactId> - <version>2.3.2</version> + <version>3.1</version> </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/spark/blob/856c50f5/docs/building-with-maven.md ---------------------------------------------------------------------- diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index 730a6e7..9cebaf1 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -6,7 +6,7 @@ title: Building Spark with Maven * This will become a table of contents (this text will be scraped). {:toc} -Building Spark using Maven Requires Maven 3 (the build process is tested with Maven 3.0.4) and Java 1.6 or newer. +Building Spark using Maven requires Maven 3.0.4 or newer and Java 1.6 or newer. ## Setting up Maven's Memory Usage ## http://git-wip-us.apache.org/repos/asf/spark/blob/856c50f5/examples/pom.xml ---------------------------------------------------------------------- diff --git a/examples/pom.xml b/examples/pom.xml index a5569ff..0b6212b 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -110,7 +110,7 @@ <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase</artifactId> - <version>0.94.6</version> + <version>${hbase.version}</version> <exclusions> <exclusion> <groupId>asm</groupId> http://git-wip-us.apache.org/repos/asf/spark/blob/856c50f5/graphx/pom.xml ---------------------------------------------------------------------- diff --git a/graphx/pom.xml b/graphx/pom.xml index 5a50229..b4c67dd 100644 --- a/graphx/pom.xml +++ b/graphx/pom.xml @@ -54,7 +54,7 @@ <dependency> <groupId>org.jblas</groupId> <artifactId>jblas</artifactId> - <version>1.2.3</version> + <version>${jblas.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> http://git-wip-us.apache.org/repos/asf/spark/blob/856c50f5/mllib/pom.xml ---------------------------------------------------------------------- diff --git a/mllib/pom.xml b/mllib/pom.xml index fec1cc9..e7ce00e 100644 --- a/mllib/pom.xml +++ b/mllib/pom.xml @@ -58,7 +58,7 @@ <dependency> <groupId>org.jblas</groupId> <artifactId>jblas</artifactId> - <version>1.2.3</version> + <version>${jblas.version}</version> </dependency> <dependency> <groupId>org.scalanlp</groupId> http://git-wip-us.apache.org/repos/asf/spark/blob/856c50f5/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 01341d2..1426e0e 100644 --- a/pom.xml +++ b/pom.xml @@ -54,11 +54,11 @@ </developers> <issueManagement> <system>JIRA</system> - <url>https://spark-project.atlassian.net/browse/SPARK</url> + <url>https://issues.apache.org/jira/browse/SPARK</url> </issueManagement> <prerequisites> - <maven>3.0.0</maven> + <maven>3.0.4</maven> </prerequisites> <mailingLists> @@ -123,6 +123,10 @@ <hbase.version>0.94.6</hbase.version> <hive.version>0.12.0</hive.version> <parquet.version>1.3.2</parquet.version> + <jblas.version>1.2.3</jblas.version> + <jetty.version>8.1.14.v20131031</jetty.version> + <chill.version>0.3.1</chill.version> + <codahale.metrics.version>3.0.0</codahale.metrics.version> <PermGen>64m</PermGen> <MaxPermGen>512m</MaxPermGen> @@ -192,22 +196,22 @@ <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util</artifactId> - <version>8.1.14.v20131031</version> + <version>${jetty.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-security</artifactId> - <version>8.1.14.v20131031</version> + <version>${jetty.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-plus</artifactId> - <version>8.1.14.v20131031</version> + <version>${jetty.version}</version> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> - <version>8.1.14.v20131031</version> + <version>${jetty.version}</version> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -273,7 +277,7 @@ <dependency> <groupId>com.twitter</groupId> <artifactId>chill_${scala.binary.version}</artifactId> - <version>0.3.1</version> + <version>${chill.version}</version> <exclusions> <exclusion> <groupId>org.ow2.asm</groupId> @@ -288,7 +292,7 @@ <dependency> <groupId>com.twitter</groupId> <artifactId>chill-java</artifactId> - <version>0.3.1</version> + <version>${chill.version}</version> <exclusions> <exclusion> <groupId>org.ow2.asm</groupId> @@ -392,27 +396,27 @@ <dependency> <groupId>com.codahale.metrics</groupId> <artifactId>metrics-core</artifactId> - <version>3.0.0</version> + <version>${codahale.metrics.version}</version> </dependency> <dependency> <groupId>com.codahale.metrics</groupId> <artifactId>metrics-jvm</artifactId> - <version>3.0.0</version> + <version>${codahale.metrics.version}</version> </dependency> <dependency> <groupId>com.codahale.metrics</groupId> <artifactId>metrics-json</artifactId> - <version>3.0.0</version> + <version>${codahale.metrics.version}</version> </dependency> <dependency> <groupId>com.codahale.metrics</groupId> <artifactId>metrics-ganglia</artifactId> - <version>3.0.0</version> + <version>${codahale.metrics.version}</version> </dependency> <dependency> <groupId>com.codahale.metrics</groupId> <artifactId>metrics-graphite</artifactId> - <version>3.0.0</version> + <version>${codahale.metrics.version}</version> </dependency> <dependency> <groupId>org.scala-lang</groupId> @@ -585,7 +589,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> - <version>1.1.1</version> + <version>1.3.1</version> <executions> <execution> <id>enforce-versions</id> @@ -595,7 +599,7 @@ <configuration> <rules> <requireMavenVersion> - <version>3.0.0</version> + <version>3.0.4</version> </requireMavenVersion> <requireJavaVersion> <version>${java.version}</version> @@ -608,12 +612,12 @@ <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>build-helper-maven-plugin</artifactId> - <version>1.7</version> + <version>1.8</version> </plugin> <plugin> <groupId>net.alchim31.maven</groupId> <artifactId>scala-maven-plugin</artifactId> - <version>3.1.5</version> + <version>3.1.6</version> <executions> <execution> <id>scala-compile-first</id> @@ -674,7 +678,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> - <version>2.12.4</version> + <version>2.17</version> <configuration> <!-- Uses scalatest instead --> <skipTests>true</skipTests> @@ -713,7 +717,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-shade-plugin</artifactId> - <version>2.0</version> + <version>2.2</version> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> @@ -810,7 +814,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> - <version>2.4</version> <executions> <execution> <goals> http://git-wip-us.apache.org/repos/asf/spark/blob/856c50f5/project/SparkBuild.scala ---------------------------------------------------------------------- diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 843a874..3489b43 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -248,10 +248,10 @@ object SparkBuild extends Build { libraryDependencies ++= Seq( "io.netty" % "netty-all" % "4.0.17.Final", - "org.eclipse.jetty" % "jetty-server" % "8.1.14.v20131031", - "org.eclipse.jetty" % "jetty-util" % "8.1.14.v20131031", - "org.eclipse.jetty" % "jetty-plus" % "8.1.14.v20131031", - "org.eclipse.jetty" % "jetty-security" % "8.1.14.v20131031", + "org.eclipse.jetty" % "jetty-server" % jettyVersion, + "org.eclipse.jetty" % "jetty-util" % jettyVersion, + "org.eclipse.jetty" % "jetty-plus" % jettyVersion, + "org.eclipse.jetty" % "jetty-security" % jettyVersion, /** Workaround for SPARK-959. Dependency used by org.eclipse.jetty. Fixed in ivy 2.3.0. */ "org.eclipse.jetty.orbit" % "javax.servlet" % "3.0.0.v201112011016" artifacts Artifact("javax.servlet", "jar", "jar"), "org.scalatest" %% "scalatest" % "1.9.1" % "test", @@ -276,6 +276,13 @@ object SparkBuild extends Build { publishLocalBoth <<= Seq(publishLocal in MavenCompile, publishLocal).dependOn ) ++ net.virtualvoid.sbt.graph.Plugin.graphSettings ++ ScalaStyleSettings + val akkaVersion = "2.2.3-shaded-protobuf" + val chillVersion = "0.3.1" + val codahaleMetricsVersion = "3.0.0" + val jblasVersion = "1.2.3" + val jettyVersion = "8.1.14.v20131031" + val hiveVersion = "0.12.0" + val parquetVersion = "1.3.2" val slf4jVersion = "1.7.5" val excludeNetty = ExclusionRule(organization = "org.jboss.netty") @@ -309,9 +316,9 @@ object SparkBuild extends Build { "commons-daemon" % "commons-daemon" % "1.0.10", // workaround for bug HADOOP-9407 "com.ning" % "compress-lzf" % "1.0.0", "org.xerial.snappy" % "snappy-java" % "1.0.5", - "org.spark-project.akka" %% "akka-remote" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty), - "org.spark-project.akka" %% "akka-slf4j" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty), - "org.spark-project.akka" %% "akka-testkit" % "2.2.3-shaded-protobuf" % "test", + "org.spark-project.akka" %% "akka-remote" % akkaVersion excludeAll(excludeNetty), + "org.spark-project.akka" %% "akka-slf4j" % akkaVersion excludeAll(excludeNetty), + "org.spark-project.akka" %% "akka-testkit" % akkaVersion % "test", "org.json4s" %% "json4s-jackson" % "3.2.6" excludeAll(excludeScalap), "it.unimi.dsi" % "fastutil" % "6.4.4", "colt" % "colt" % "1.2.0", @@ -321,12 +328,12 @@ object SparkBuild extends Build { "org.apache.derby" % "derby" % "10.4.2.0" % "test", "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm), "org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty), - "com.codahale.metrics" % "metrics-core" % "3.0.0", - "com.codahale.metrics" % "metrics-jvm" % "3.0.0", - "com.codahale.metrics" % "metrics-json" % "3.0.0", - "com.codahale.metrics" % "metrics-graphite" % "3.0.0", - "com.twitter" %% "chill" % "0.3.1" excludeAll(excludeAsm), - "com.twitter" % "chill-java" % "0.3.1" excludeAll(excludeAsm), + "com.codahale.metrics" % "metrics-core" % codahaleMetricsVersion, + "com.codahale.metrics" % "metrics-jvm" % codahaleMetricsVersion, + "com.codahale.metrics" % "metrics-json" % codahaleMetricsVersion, + "com.codahale.metrics" % "metrics-graphite" % codahaleMetricsVersion, + "com.twitter" %% "chill" % chillVersion excludeAll(excludeAsm), + "com.twitter" % "chill-java" % chillVersion excludeAll(excludeAsm), "org.tachyonproject" % "tachyon" % "0.4.1-thrift" excludeAll(excludeHadoop, excludeCurator, excludeEclipseJetty, excludePowermock), "com.clearspring.analytics" % "stream" % "2.5.1" ), @@ -370,7 +377,7 @@ object SparkBuild extends Build { name := "spark-graphx", previousArtifact := sparkPreviousArtifact("spark-graphx"), libraryDependencies ++= Seq( - "org.jblas" % "jblas" % "1.2.3" + "org.jblas" % "jblas" % jblasVersion ) ) @@ -383,7 +390,7 @@ object SparkBuild extends Build { name := "spark-mllib", previousArtifact := sparkPreviousArtifact("spark-mllib"), libraryDependencies ++= Seq( - "org.jblas" % "jblas" % "1.2.3", + "org.jblas" % "jblas" % jblasVersion, "org.scalanlp" %% "breeze" % "0.7" ) ) @@ -403,8 +410,8 @@ object SparkBuild extends Build { def sqlCoreSettings = sharedSettings ++ Seq( name := "spark-sql", libraryDependencies ++= Seq( - "com.twitter" % "parquet-column" % "1.3.2", - "com.twitter" % "parquet-hadoop" % "1.3.2" + "com.twitter" % "parquet-column" % parquetVersion, + "com.twitter" % "parquet-hadoop" % parquetVersion ) ) @@ -416,9 +423,9 @@ object SparkBuild extends Build { jarName in packageDependency <<= version map { v => "spark-hive-assembly-" + v + "-hadoop" + hadoopVersion + "-deps.jar" }, javaOptions += "-XX:MaxPermSize=1g", libraryDependencies ++= Seq( - "org.apache.hive" % "hive-metastore" % "0.12.0", - "org.apache.hive" % "hive-exec" % "0.12.0", - "org.apache.hive" % "hive-serde" % "0.12.0" + "org.apache.hive" % "hive-metastore" % hiveVersion, + "org.apache.hive" % "hive-exec" % hiveVersion, + "org.apache.hive" % "hive-serde" % hiveVersion ), // Multiple queries rely on the TestHive singleton. See comments there for more details. parallelExecution in Test := false, @@ -549,7 +556,7 @@ object SparkBuild extends Build { name := "spark-streaming-zeromq", previousArtifact := sparkPreviousArtifact("spark-streaming-zeromq"), libraryDependencies ++= Seq( - "org.spark-project.akka" %% "akka-zeromq" % "2.2.3-shaded-protobuf" excludeAll(excludeNetty) + "org.spark-project.akka" %% "akka-zeromq" % akkaVersion excludeAll(excludeNetty) ) ) http://git-wip-us.apache.org/repos/asf/spark/blob/856c50f5/streaming/pom.xml ---------------------------------------------------------------------- diff --git a/streaming/pom.xml b/streaming/pom.xml index 1953cc6..93b1c5a 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -96,7 +96,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-jar-plugin</artifactId> - <version>2.2</version> <executions> <execution> <goals>
