Repository: bigtop Updated Branches: refs/heads/master 9dd5f5a1d -> 9e78a0365
BIGTOP-2959: Add SPARK_DIST_CLASSPATH to include libraries for HDFS and YARN Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/9e78a036 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/9e78a036 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/9e78a036 Branch: refs/heads/master Commit: 9e78a03655ee68b8a60258c2a5efd9dbe6dfac30 Parents: 9dd5f5a Author: Youngwoo Kim <yw...@apache.org> Authored: Tue Dec 26 15:17:19 2017 +0900 Committer: Youngwoo Kim <yw...@apache.org> Committed: Wed Dec 27 11:24:13 2017 +0900 ---------------------------------------------------------------------- .../src/common/spark/do-component-build | 2 +- bigtop-packages/src/common/spark/spark-env.sh | 3 +-- bigtop-packages/src/deb/spark/rules | 17 ----------------- bigtop-packages/src/rpm/spark/SPECS/spark.spec | 17 ----------------- 4 files changed, 2 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/common/spark/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/do-component-build b/bigtop-packages/src/common/spark/do-component-build index c2a2322..084f9f2 100644 --- a/bigtop-packages/src/common/spark/do-component-build +++ b/bigtop-packages/src/common/spark/do-component-build @@ -28,7 +28,7 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${ -Dyarn.version=$HADOOP_VERSION \ -Dprotobuf.version=2.5.0 \ -DrecompileMode=all \ - -Pyarn -Phadoop-2.7 \ + -Pyarn -Phadoop-2.7 -Phadoop-provided \ -Phive -Phive-thriftserver \ -Psparkr \ $SPARK_BUILD_OPTS" http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/common/spark/spark-env.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/spark/spark-env.sh b/bigtop-packages/src/common/spark/spark-env.sh index f772ef1..1806119 100755 --- a/bigtop-packages/src/common/spark/spark-env.sh +++ b/bigtop-packages/src/common/spark/spark-env.sh @@ -28,8 +28,7 @@ export SPARK_LAUNCH_WITH_SCALA=0 export SPARK_LIBRARY_PATH=${SPARK_LIBRARY_PATH:-${SPARK_HOME}/lib} export SCALA_LIBRARY_PATH=${SCALA_LIBRARY_PATH:-${SPARK_HOME}/lib} -# Let's make sure that all needed hadoop libs are added properly -export CLASSPATH="$CLASSPATH:$HADOOP_HOME/*:$HADOOP_HDFS_HOME/*:$HADOOP_YARN_HOME/*:$HADOOP_MAPRED_HOME/*" +export SPARK_DIST_CLASSPATH=$(hadoop classpath) export SPARK_LIBRARY_PATH=$SPARK_LIBRARY_PATH:${HADOOP_HOME}/lib/native export STANDALONE_SPARK_MASTER_HOST=`hostname -f` http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/deb/spark/rules ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/deb/spark/rules b/bigtop-packages/src/deb/spark/rules index d5fdb6d..5bd9833 100644 --- a/bigtop-packages/src/deb/spark/rules +++ b/bigtop-packages/src/deb/spark/rules @@ -49,20 +49,3 @@ override_dh_install: dh_install # BIGTOP-2588 drop datanucleus jars from spark-core rm -Rf debian/spark-core/usr/lib/spark/jars/datanucleus* - - rm -f debian/tmp/${lib_spark}/jars/hadoop-*.jar - ln -s ${lib_hadoop_client}/hadoop-annotations.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_client}/hadoop-auth.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_client}/hadoop-client.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_client}/hadoop-common.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_client}/hadoop-hdfs.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_client}/hadoop-mapreduce-client-app.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_client}/hadoop-mapreduce-client-common.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_client}/hadoop-mapreduce-client-core.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_client}/hadoop-mapreduce-client-jobclient.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_client}/hadoop-mapreduce-client-shuffle.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_yarn}/hadoop-yarn-api.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_yarn}/hadoop-yarn-client.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_yarn}/hadoop-yarn-common.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_yarn}/hadoop-yarn-server-common.jar debian/tmp/${lib_spark}/jars/ - ln -s ${lib_hadoop_yarn}/hadoop-yarn-server-web-proxy.jar debian/tmp/${lib_spark}/jars/ http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/rpm/spark/SPECS/spark.spec ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec index c29c68a..8e73cd8 100644 --- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec +++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec @@ -166,23 +166,6 @@ bash $RPM_SOURCE_DIR/install_spark.sh \ --prefix=$RPM_BUILD_ROOT \ --doc-dir=%{doc_spark} -%__rm -f $RPM_BUILD_ROOT/%{lib_spark}/jars/hadoop-*.jar -%__ln_s %{lib_hadoop_client}/hadoop-annotations.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_client}/hadoop-auth.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_client}/hadoop-client.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_client}/hadoop-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_client}/hadoop-hdfs.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_client}/hadoop-mapreduce-client-app.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_client}/hadoop-mapreduce-client-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_client}/hadoop-mapreduce-client-core.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_client}/hadoop-mapreduce-client-jobclient.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_client}/hadoop-mapreduce-client-shuffle.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_yarn}/hadoop-yarn-api.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_yarn}/hadoop-yarn-client.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_yarn}/hadoop-yarn-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_yarn}/hadoop-yarn-server-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ -%__ln_s %{lib_hadoop_yarn}/hadoop-yarn-server-web-proxy.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/ - for service in %{spark_services} do # Install init script
