Repository: spark Updated Branches: refs/heads/branch-1.0 e88a63625 -> acbf307c2
The default version of yarn is equal to the hadoop version This is a part of [PR 590](https://github.com/apache/spark/pull/590) Author: witgo <wi...@qq.com> Closes #626 from witgo/yarn_version and squashes the following commits: c390631 [witgo] restore the yarn dependency declarations f8a4ad8 [witgo] revert remove the dependency of avro in yarn-alpha 2df6cf5 [witgo] review commit a1d876a [witgo] review commit 20e7e3e [witgo] review commit c76763b [witgo] The default value of yarn.version is equal to hadoop.version (cherry picked from commit fb0543224bcedb8ae3aab4a7ddcc6111a03378fe) Signed-off-by: Patrick Wendell <pwend...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/acbf307c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/acbf307c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/acbf307c Branch: refs/heads/branch-1.0 Commit: acbf307c25ca681ffaee3a9d9acaefb56a2f11c5 Parents: e88a636 Author: witgo <wi...@qq.com> Authored: Sat May 3 23:32:12 2014 -0700 Committer: Patrick Wendell <pwend...@gmail.com> Committed: Sat May 3 23:32:24 2014 -0700 ---------------------------------------------------------------------- bin/compute-classpath.sh | 1 + docs/building-with-maven.md | 9 ++++++--- pom.xml | 15 +++++++++------ yarn/pom.xml | 5 ++--- 4 files changed, 18 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/acbf307c/bin/compute-classpath.sh ---------------------------------------------------------------------- diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index b021853..70ac71e 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -44,6 +44,7 @@ if [ -f "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar ]; then CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SCALA_VERSION/classes" + CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SCALA_VERSION/classes" DEPS_ASSEMBLY_JAR=`ls "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar` CLASSPATH="$CLASSPATH:$DEPS_ASSEMBLY_JAR" http://git-wip-us.apache.org/repos/asf/spark/blob/acbf307c/docs/building-with-maven.md ---------------------------------------------------------------------- diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index a5e5303..e447dfe 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -45,17 +45,20 @@ For Apache Hadoop versions 1.x, Cloudera CDH MRv1, and other Hadoop versions wit For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with YARN, you can enable the "yarn-alpha" or "yarn" profile and set the "hadoop.version", "yarn.version" property. Note that Hadoop 0.23.X requires a special `-Phadoop-0.23` profile: # Apache Hadoop 2.0.5-alpha - $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -Dyarn.version=2.0.5-alpha -DskipTests clean package + $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -DskipTests clean package # Cloudera CDH 4.2.0 with MapReduce v2 - $ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -Dyarn.version=2.0.0-cdh4.2.0 -DskipTests clean package + $ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -DskipTests clean package # Apache Hadoop 2.2.X (e.g. 2.2.0 as below) and newer - $ mvn -Pyarn -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 -DskipTests clean package + $ mvn -Pyarn -Dhadoop.version=2.2.0 -DskipTests clean package # Apache Hadoop 0.23.x $ mvn -Pyarn-alpha -Phadoop-0.23 -Dhadoop.version=0.23.7 -Dyarn.version=0.23.7 -DskipTests clean package + # Different versions of HDFS and YARN. + $ mvn -Pyarn-alpha -Dhadoop.version=2.3.0 -Dyarn.version=0.23.7 -DskipTests clean package + ## Spark Tests in Maven ## Tests are run by default via the [ScalaTest Maven plugin](http://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin). Some of the require Spark to be packaged first, so always run `mvn package` with `-DskipTests` the first time. You can then run the tests with `mvn -Dhadoop.version=... test`. http://git-wip-us.apache.org/repos/asf/spark/blob/acbf307c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 1075bb5..2a4d595 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,8 @@ ~ limitations under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache</groupId> @@ -119,7 +120,7 @@ <log4j.version>1.2.17</log4j.version> <hadoop.version>1.0.4</hadoop.version> <protobuf.version>2.4.1</protobuf.version> - <yarn.version>0.23.7</yarn.version> + <yarn.version>${hadoop.version}</yarn.version> <hbase.version>0.94.6</hbase.version> <hive.version>0.12.0</hive.version> <parquet.version>1.3.2</parquet.version> @@ -135,7 +136,8 @@ <repositories> <repository> - <id>maven-repo</id> <!-- This should be at top, it makes maven try the central repo first and then others and hence faster dep resolution --> + <id>maven-repo</id> + <!-- This should be at top, it makes maven try the central repo first and then others and hence faster dep resolution --> <name>Maven Repository</name> <!-- HTTPS is unavailable for Maven Central --> <url>http://repo.maven.apache.org/maven2</url> @@ -847,15 +849,16 @@ <hadoop.version>0.23.7</hadoop.version> <!--<hadoop.version>2.0.5-alpha</hadoop.version> --> </properties> - <modules> - <module>yarn</module> - </modules> <dependencies> <dependency> <groupId>org.apache.avro</groupId> <artifactId>avro</artifactId> </dependency> </dependencies> + <modules> + <module>yarn</module> + </modules> + </profile> <!-- Ganglia integration is not included by default due to LGPL-licensed code --> http://git-wip-us.apache.org/repos/asf/spark/blob/acbf307c/yarn/pom.xml ---------------------------------------------------------------------- diff --git a/yarn/pom.xml b/yarn/pom.xml index c8163f5..62558fc 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -28,7 +28,7 @@ <artifactId>yarn-parent_2.10</artifactId> <packaging>pom</packaging> <name>Spark Project YARN Parent POM</name> - + <dependencies> <dependency> <groupId>org.apache.spark</groupId> @@ -50,7 +50,6 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> - <version>${yarn.version}</version> </dependency> <dependency> <groupId>org.scalatest</groupId> @@ -128,7 +127,7 @@ <target> <property name="spark.classpath" refid="maven.test.classpath" /> <property environment="env" /> - <fail message="Please set the SCALA_HOME (or SCALA_LIBRARY_PATH if scala is on the path) environment variables and retry."> + <fail message="Please set the SCALA_HOME (or SCALA_LIBRARY_PATH if scala is on the path) environment variables and retry."> <condition> <not> <or>