LENS-515 : Support to run spark in yarn-client mode in lens-ml module
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/44a6f516 Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/44a6f516 Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/44a6f516 Branch: refs/heads/current-release-line Commit: 44a6f5162858e243968280f2fda27ae614f36e4e Parents: 37d2fc1 Author: Sharad Agarwal <[email protected]> Authored: Wed Jun 17 16:50:54 2015 +0530 Committer: Amareshwari Sriramadasu <[email protected]> Committed: Wed Jun 17 16:50:54 2015 +0530 ---------------------------------------------------------------------- lens-docker/lens-test/lens-bootstrap.sh | 6 +++++- lens-ml-dist/pom.xml | 20 ++++++++++++++++++++ lens-ml-lib/pom.xml | 4 ++++ .../lens/ml/algo/spark/SparkMLDriver.java | 7 +++++++ pom.xml | 5 +++++ tools/conf-pseudo-distr/server/lens-site.xml | 2 +- tools/scripts/lens-ctl | 4 +++- 7 files changed, 45 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-docker/lens-test/lens-bootstrap.sh ---------------------------------------------------------------------- diff --git a/lens-docker/lens-test/lens-bootstrap.sh b/lens-docker/lens-test/lens-bootstrap.sh index 11cf1b9..3407dca 100644 --- a/lens-docker/lens-test/lens-bootstrap.sh +++ b/lens-docker/lens-test/lens-bootstrap.sh @@ -28,7 +28,11 @@ echo "SPARK_HOME " $SPARK_HOME LENS_EXT_CLASSPATH=$LENS_EXT_CLASSPATH:`$LENS_ML/bin/lens-ml-classpath.sh` export LENS_EXT_CLASSPATH -HIVE_AUX_JARS_PATH=$LENS_ML/lib/lens-ml-lib-2.2.0-beta-incubating-SNAPSHOT.jar,$SPARK_HOME/lib/spark-assembly-1.3.0-hadoop2.4.0.jar +SPARK_YARN_JAR=$SPARK_HOME/lib/spark-assembly-1.3.0-hadoop2.4.0.jar +export SPARK_YARN_JAR +echo "SPARK_YARN_JAR " $SPARK_YARN_JAR + +HIVE_AUX_JARS_PATH=$LENS_ML/lib/lens-ml-lib-2.2.0-beta-incubating-SNAPSHOT.jar,$SPARK_YARN_JAR export HIVE_AUX_JARS_PATH echo "HIVE_AUX_JARS_PATH " $HIVE_AUX_JARS_PATH http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-ml-dist/pom.xml ---------------------------------------------------------------------- diff --git a/lens-ml-dist/pom.xml b/lens-ml-dist/pom.xml index 54c9e51..26be98d 100644 --- a/lens-ml-dist/pom.xml +++ b/lens-ml-dist/pom.xml @@ -56,6 +56,26 @@ <artifactId>hadoop-common</artifactId> </exclusion> <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-common</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-web-proxy</artifactId> + </exclusion> + <exclusion> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> </exclusion> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-ml-lib/pom.xml ---------------------------------------------------------------------- diff --git a/lens-ml-lib/pom.xml b/lens-ml-lib/pom.xml index f952f8a..63d72da 100644 --- a/lens-ml-lib/pom.xml +++ b/lens-ml-lib/pom.xml @@ -88,6 +88,10 @@ <groupId>org.apache.spark</groupId> <artifactId>spark-mllib_2.10</artifactId> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-yarn_2.10</artifactId> + </dependency> <dependency> <groupId>org.apache.hive.hcatalog</groupId> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java ---------------------------------------------------------------------- diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java index 6543fed..21ed87d 100644 --- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java +++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java @@ -172,7 +172,14 @@ public class SparkMLDriver implements MLDriver { throw new IllegalArgumentException("Spark home is not set"); } + // set spark.yarn.jar + String yarnJars = System.getenv("SPARK_YARN_JAR"); + if (StringUtils.isNotBlank(yarnJars)) { + sparkConf.set("spark.yarn.jar", yarnJars); + } + LOG.info("Spark home is set to " + sparkConf.get("spark.home")); + LOG.info("spark.yarn.jar is set to " + yarnJars); } sparkConf.setAppName("lens-ml"); http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 067f9b4..df0b766 100644 --- a/pom.xml +++ b/pom.xml @@ -1165,6 +1165,11 @@ <artifactId>spark-mllib_2.10</artifactId> <version>${spark.version}</version> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-yarn_2.10</artifactId> + <version>${spark.version}</version> + </dependency> <dependency> <groupId>org.apache.hive.hcatalog</groupId> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/tools/conf-pseudo-distr/server/lens-site.xml ---------------------------------------------------------------------- diff --git a/tools/conf-pseudo-distr/server/lens-site.xml b/tools/conf-pseudo-distr/server/lens-site.xml index 6799987..f43d07e 100644 --- a/tools/conf-pseudo-distr/server/lens-site.xml +++ b/tools/conf-pseudo-distr/server/lens-site.xml @@ -76,7 +76,7 @@ <property> <name>lens.ml.sparkdriver.spark.master</name> - <value>local</value> + <value>yarn-client</value> </property> <property> http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/tools/scripts/lens-ctl ---------------------------------------------------------------------- diff --git a/tools/scripts/lens-ctl b/tools/scripts/lens-ctl index ec6b5fe..1335469 100755 --- a/tools/scripts/lens-ctl +++ b/tools/scripts/lens-ctl @@ -116,7 +116,9 @@ start() { MAPRED_JARS=$MAPRED_JARS:`ls ${HADOOP_HOME}/../hadoop-mapreduce/aws-java-sdk-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null` MAPRED_JARS=$MAPRED_JARS:`ls ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null` - HADOOP_JARPATH=$CORE_JARS:$LIB_JARS:$COMMON_JARS:$HDFS_JARS:$MAPRED_JARS + YARN_JARS=`ls ${HADOOP_HOME}/share/hadoop/yarn/hadoop-yarn*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null` + + HADOOP_JARPATH=$CORE_JARS:$LIB_JARS:$COMMON_JARS:$HDFS_JARS:$MAPRED_JARS:$YARN_JARS LENSCPPATH=${LENSCPPATH}:$HADOOP_JARPATH fi
