Repository: incubator-zeppelin Updated Branches: refs/heads/master a3a86f883 -> 58fd82b86
[ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Removed SPARK_YARN_JAR dependencies Author: Jongyoul Lee <[email protected]> Closes #4 from jongyoul/ZEPPELIN-7 and squashes the following commits: df697c1 [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Updated README.md 7f84a5a [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Reverted to initial commit b366d16 [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Fixed 2 spaces indentations 29ea46b [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Removed unused codes 91066c4 [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Fixed errors while running 32be4ca [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Added maven-shade-plugin - Removed copy-dependencies Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/58fd82b8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/58fd82b8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/58fd82b8 Branch: refs/heads/master Commit: 58fd82b86207cc773ced5624a387705ba02aa576 Parents: a3a86f8 Author: Jongyoul Lee <[email protected]> Authored: Fri Mar 27 11:34:40 2015 +0900 Committer: Lee moon soo <[email protected]> Committed: Mon Mar 30 16:55:23 2015 +0900 ---------------------------------------------------------------------- README.md | 4 +- spark/pom.xml | 112 ++++++++++++++++++++++++++++++----------------------- 2 files changed, 65 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/58fd82b8/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 93df312..46e188c 100644 --- a/README.md +++ b/README.md @@ -85,10 +85,8 @@ If you set `SPARK_HOME`, you should deploy spark binary on the same location to Yarn # ./conf/zeppelin-env.sh - export SPARK_YARN_JAR=/path/to/spark-assembly-*.jar export HADOOP_CONF_DIR=/path/to/hadoop_conf_dir - -`SPARK_YARN_JAR` is deployed for running executor, this could be a local path or HDFS. HDFS allows YARN to cache it on nodes so that it doesn't need to be distributed each time an application runs. To point to a jar on HDFS, for example, set this configuration to "hdfs:///some/path". + `HADOOP_CONF_DIR` should contains yarn-site.xml and core-site.xml. ### Run http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/58fd82b8/spark/pom.xml ---------------------------------------------------------------------- diff --git a/spark/pom.xml b/spark/pom.xml index 49b4d08..fcecc1b 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -1,5 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <modelVersion>4.0.0</modelVersion> @@ -9,7 +10,6 @@ <version>0.5.0-SNAPSHOT</version> </parent> - <groupId>com.nflabs.zeppelin</groupId> <artifactId>zeppelin-spark</artifactId> <packaging>jar</packaging> <version>0.5.0-SNAPSHOT</version> @@ -67,7 +67,7 @@ <groupId>org.scala-lang</groupId> <artifactId>scala-library</artifactId> </dependency> - + <dependency> <groupId>org.scala-lang</groupId> <artifactId>scala-compiler</artifactId> @@ -77,7 +77,7 @@ <groupId>org.scala-lang</groupId> <artifactId>scala-reflect</artifactId> </dependency> - + <dependency> <groupId>org.eclipse.jetty.aggregate</groupId> <artifactId>jetty-all-server</artifactId> @@ -94,27 +94,27 @@ <version>${spark.version}</version> </dependency> - <dependency> + <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-sql_2.10</artifactId> <version>${spark.version}</version> </dependency> - <dependency> + <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-hive_2.10</artifactId> <version>${spark.version}</version> </dependency> - <dependency> + <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming_2.10</artifactId> <version>${spark.version}</version> </dependency> - <dependency> + <dependency> <groupId>org.apache.spark</groupId> <artifactId>spark-streaming-twitter_2.10</artifactId> <version>${spark.version}</version> @@ -125,7 +125,7 @@ <artifactId>spark-catalyst_${scala.binary.version}</artifactId> <version>${spark.version}</version> </dependency> - + <!-- Aether :: maven dependency resolution --> <dependency> <groupId>org.apache.maven</groupId> @@ -139,12 +139,12 @@ <exclusion> <groupId>org.sonatype.sisu</groupId> <artifactId>sisu-inject-plexus</artifactId> - </exclusion> + </exclusion> <exclusion> <groupId>org.apache.maven</groupId> <artifactId>maven-model</artifactId> - </exclusion> - </exclusions> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.sonatype.aether</groupId> @@ -161,7 +161,7 @@ <artifactId>aether-impl</artifactId> <version>1.12</version> </dependency> - + <dependency> <groupId>org.apache.maven</groupId> <artifactId>maven-aether-provider</artifactId> @@ -170,7 +170,7 @@ <exclusion> <groupId>org.sonatype.aether</groupId> <artifactId>aether-api</artifactId> - </exclusion> + </exclusion> <exclusion> <groupId>org.sonatype.aether</groupId> <artifactId>aether-spi</artifactId> @@ -178,24 +178,24 @@ <exclusion> <groupId>org.sonatype.aether</groupId> <artifactId>aether-util</artifactId> - </exclusion> + </exclusion> <exclusion> <groupId>org.sonatype.aether</groupId> <artifactId>aether-impl</artifactId> - </exclusion> + </exclusion> <exclusion> <groupId>org.codehaus.plexus</groupId> <artifactId>plexus-utils</artifactId> - </exclusion> - </exclusions> + </exclusion> + </exclusions> </dependency> - + <dependency> <groupId>org.sonatype.aether</groupId> <artifactId>aether-connector-file</artifactId> <version>1.12</version> </dependency> - + <dependency> <groupId>org.sonatype.aether</groupId> <artifactId>aether-connector-wagon</artifactId> @@ -205,9 +205,9 @@ <groupId>org.apache.maven.wagon</groupId> <artifactId>wagon-provider-api</artifactId> </exclusion> - </exclusions> + </exclusions> </dependency> - + <dependency> <groupId>org.apache.maven.wagon</groupId> <artifactId>wagon-provider-api</artifactId> @@ -216,10 +216,10 @@ <exclusion> <groupId>org.codehaus.plexus</groupId> <artifactId>plexus-utils</artifactId> - </exclusion> - </exclusions> + </exclusion> + </exclusions> </dependency> - + <dependency> <groupId>org.apache.maven.wagon</groupId> <artifactId>wagon-http-lightweight</artifactId> @@ -230,7 +230,7 @@ <artifactId>wagon-http-shared</artifactId> </exclusion> </exclusions> - </dependency> + </dependency> <dependency> <groupId>org.apache.maven.wagon</groupId> @@ -259,7 +259,7 @@ <scope>test</scope> </dependency> </dependencies> - + <profiles> <profile> <id>yarn</id> @@ -272,7 +272,7 @@ </dependencies> </profile> </profiles> - + <build> <plugins> <plugin> @@ -286,11 +286,11 @@ <plugin> <artifactId>maven-enforcer-plugin</artifactId> - <version>1.3.1</version> - <executions> - <execution> - <id>enforce</id> - <phase>none</phase> + <version>1.3.1</version> + <executions> + <execution> + <id>enforce</id> + <phase>none</phase> </execution> </executions> </plugin> @@ -307,25 +307,42 @@ </plugin> <plugin> - <artifactId>maven-dependency-plugin</artifactId> - <version>2.8</version> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.3</version> + <configuration> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> + <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> + <resource>reference.conf</resource> + </transformer> + </transformers> + </configuration> <executions> <execution> - <id>copy-dependencies</id> <phase>package</phase> <goals> - <goal>copy-dependencies</goal> + <goal>shade</goal> </goals> - <configuration> - <outputDirectory>${project.build.directory}/../../interpreter/spark</outputDirectory> - <overWriteReleases>false</overWriteReleases> - <overWriteSnapshots>false</overWriteSnapshots> - <overWriteIfNewer>true</overWriteIfNewer> - <includeScope>runtime</includeScope> - </configuration> </execution> + </executions> + </plugin> + + <plugin> + <artifactId>maven-dependency-plugin</artifactId> + <version>2.8</version> + <executions> <execution> - <id>copy-artifact</id> <phase>package</phase> <goals> <goal>copy</goal> @@ -335,7 +352,6 @@ <overWriteReleases>false</overWriteReleases> <overWriteSnapshots>false</overWriteSnapshots> <overWriteIfNewer>true</overWriteIfNewer> - <includeScope>runtime</includeScope> <artifactItems> <artifactItem> <groupId>${project.groupId}</groupId> @@ -343,11 +359,11 @@ <version>${project.version}</version> <type>${project.packaging}</type> </artifactItem> - </artifactItems> + </artifactItems> </configuration> </execution> </executions> </plugin> </plugins> - </build> + </build> </project>
