Repository: zeppelin Updated Branches: refs/heads/master 52202afd3 -> d2f63f6f9
[ZEPPELIN-1520] Change test environment to use the local mode of Apache Spark ### What is this PR for? Simplify travis test to reduce usage of resources ### What type of PR is it? [Improvement] ### Todos * [x] - Remove start-up/stop SparkCluster ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-1520 ### How should this be tested? Travis will pass without any error ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Jongyoul Lee <jongy...@gmail.com> Closes #1487 from jongyoul/ZEPPELIN-1520 and squashes the following commits: 3bccf66 [Jongyoul Lee] Removed some unused scripts anymore 15a3711 [Jongyoul Lee] Cleaned up commented lines 1237658 [Jongyoul Lee] Removed checking mechanism f37dacf [Jongyoul Lee] Changed master to local[2] 2aac444 [Jongyoul Lee] Remove scripts of start/stop SparkCluster Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/d2f63f6f Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/d2f63f6f Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/d2f63f6f Branch: refs/heads/master Commit: d2f63f6f96f1921ebc901915abc606f431461cb8 Parents: 52202af Author: Jongyoul Lee <jongy...@gmail.com> Authored: Mon Oct 10 00:02:25 2016 +0900 Committer: Jongyoul Lee <jongy...@apache.org> Committed: Tue Oct 11 17:13:20 2016 +0900 ---------------------------------------------------------------------- .travis.yml | 4 -- testing/startSparkCluster.sh | 67 -------------------- testing/stopSparkCluster.sh | 42 ------------ .../zeppelin/rest/AbstractTestRestApi.java | 12 +--- 4 files changed, 3 insertions(+), 122 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index e2a115c..d314b2f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,7 +89,6 @@ install: before_script: - travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER - - ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER - echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh - tail conf/zeppelin-env.sh @@ -109,6 +108,3 @@ after_failure: - cat zeppelin-web/npm-debug.log - cat spark-*/logs/* -after_script: - - ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER - http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/testing/startSparkCluster.sh ---------------------------------------------------------------------- diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh deleted file mode 100755 index dc7613d..0000000 --- a/testing/startSparkCluster.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -if [[ "$#" -ne 2 ]]; then - echo "usage) $0 [spark version] [hadoop version]" - echo " eg) $0 1.3.1 2.6" - exit 1 -fi - -SPARK_VERSION="${1}" -HADOOP_VERSION="${2}" - -echo "${SPARK_VERSION}" | grep "^1.[123].[0-9]" > /dev/null -if [[ "$?" -eq 0 ]]; then - echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null - if [[ "$?" -eq 0 ]]; then - SPARK_VER_RANGE="<=1.2" - else - SPARK_VER_RANGE="<=1.3" - fi -else - SPARK_VER_RANGE=">1.3" -fi - -set -xe - -FWDIR="$(dirname "${BASH_SOURCE-$0}")" -ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" - -SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" -export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}" -echo "SPARK_HOME is ${SPARK_HOME}" - -# create PID dir. test case detect pid file so they can select active spark home dir for test -export SPARK_PID_DIR="${SPARK_HOME}/run" -mkdir -p "${SPARK_PID_DIR}" - -# start -export SPARK_MASTER_PORT=7071 -export SPARK_MASTER_WEBUI_PORT=7072 -export SPARK_WORKER_WEBUI_PORT=8082 -${SPARK_HOME}/sbin/start-master.sh - -if [ "${SPARK_VER_RANGE}" == "<=1.3" ]||[ "${SPARK_VER_RANGE}" == "<=1.2" ]; then - # spark 1.3 or prior - ${SPARK_HOME}/sbin/start-slave.sh 1 `hostname`:${SPARK_MASTER_PORT} -else - ${SPARK_HOME}/sbin/start-slave.sh spark://`hostname`:7071 -fi - -set +xe http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/testing/stopSparkCluster.sh ---------------------------------------------------------------------- diff --git a/testing/stopSparkCluster.sh b/testing/stopSparkCluster.sh deleted file mode 100755 index e049ec4..0000000 --- a/testing/stopSparkCluster.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -if [[ "$#" -ne 2 ]]; then - echo "usage) $0 [spark version] [hadoop version]" - echo " eg) $0 1.3.1 2.6" - exit 1 -fi - -SPARK_VERSION="${1}" -HADOOP_VERSION="${2}" - -set -xe - -FWDIR="$(dirname "${BASH_SOURCE-$0}")" -ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)" - -SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" -export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}" -echo "SPARK_HOME is ${SPARK_HOME}" - -# set create PID dir -export SPARK_PID_DIR="${SPARK_HOME}/run" - -${SPARK_HOME}/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1 -${SPARK_HOME}/sbin/stop-master.sh -set +xe http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index eb080fe..de2607f 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -152,7 +152,7 @@ public abstract class AbstractTestRestApi { } // set spark master and other properties - sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071"); + sparkIntpSetting.getProperties().setProperty("master", "local[2]"); sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2"); sparkIntpSetting.getProperties().setProperty("zeppelin.spark.useHiveContext", "false"); // set spark home for pyspark @@ -175,7 +175,7 @@ public abstract class AbstractTestRestApi { sparkIntpSetting.getProperties().setProperty("master", System.getenv("SPARK_MASTER")); } else { sparkIntpSetting.getProperties() - .setProperty("master", "spark://" + getHostname() + ":7071"); + .setProperty("master", "local[2]"); } sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2"); // set spark home for pyspark @@ -238,13 +238,7 @@ public abstract class AbstractTestRestApi { } private static boolean isActiveSparkHome(File dir) { - if (dir.getName().matches("spark-[0-9\\.]+[A-Za-z-]*-bin-hadoop[0-9\\.]+")) { - File pidDir = new File(dir, "run"); - if (pidDir.isDirectory() && pidDir.listFiles().length > 0) { - return true; - } - } - return false; + return dir.getName().matches("spark-[0-9\\.]+[A-Za-z-]*-bin-hadoop[0-9\\.]+"); } protected static void shutDown() throws Exception {
