Repository: spark Updated Branches: refs/heads/branch-1.0 4ff39292c -> 0c3e4150f
SPARK-1474: Spark on yarn assembly doesn't include AmIpFilter We use org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter in spark on yarn but are not included it in the assembly jar. I tested this on yarn cluster by removing the yarn jars from the classpath and spark runs fine now. Author: Thomas Graves <[email protected]> Closes #406 from tgravescs/SPARK-1474 and squashes the following commits: 1548bf9 [Thomas Graves] SPARK-1474: Spark on yarn assembly doesn't include org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter (cherry picked from commit 1e829905c791fbf1dfd8e0c1caa62ead7354605e) Signed-off-by: Patrick Wendell <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0c3e4150 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0c3e4150 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0c3e4150 Branch: refs/heads/branch-1.0 Commit: 0c3e4150f6b91f483574ef4d1b7b90d5dd69ec15 Parents: 4ff3929 Author: Thomas Graves <[email protected]> Authored: Tue May 6 12:00:09 2014 -0700 Committer: Patrick Wendell <[email protected]> Committed: Tue May 6 12:00:24 2014 -0700 ---------------------------------------------------------------------- pom.xml | 25 ++++++++++++++++++++++++- project/SparkBuild.scala | 3 ++- yarn/pom.xml | 4 ++++ 3 files changed, 30 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/0c3e4150/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3b8cb1c..513a0d0 100644 --- a/pom.xml +++ b/pom.xml @@ -616,7 +616,25 @@ </exclusion> </exclusions> </dependency> - + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-web-proxy</artifactId> + <version>${yarn.version}</version> + <exclusions> + <exclusion> + <groupId>asm</groupId> + <artifactId>asm</artifactId> + </exclusion> + <exclusion> + <groupId>org.ow2.asm</groupId> + <artifactId>asm</artifactId> + </exclusion> + <exclusion> + <groupId>org.jboss.netty</groupId> + <artifactId>netty</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> @@ -960,6 +978,11 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-web-proxy</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> <scope>provided</scope> </dependency> http://git-wip-us.apache.org/repos/asf/spark/blob/0c3e4150/project/SparkBuild.scala ---------------------------------------------------------------------- diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index e143861..9b621a3 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -563,7 +563,8 @@ object SparkBuild extends Build { "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm), "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm), "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm), - "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm) + "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm), + "org.apache.hadoop" % "hadoop-yarn-server-web-proxy" % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeOldAsm) ) ) http://git-wip-us.apache.org/repos/asf/spark/blob/0c3e4150/yarn/pom.xml ---------------------------------------------------------------------- diff --git a/yarn/pom.xml b/yarn/pom.xml index 62558fc..983ce6f 100644 --- a/yarn/pom.xml +++ b/yarn/pom.xml @@ -45,6 +45,10 @@ </dependency> <dependency> <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-web-proxy</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-client</artifactId> </dependency> <dependency>
