BIGTOP-2441. spark smoke test doesn't work relying on yarn-client submittion
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6e81da1e Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6e81da1e Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6e81da1e Branch: refs/heads/BIGTOP-2253 Commit: 6e81da1e896338927af4160854794da489718a67 Parents: b6f50e6 Author: Konstantin Boudnik <c...@apache.org> Authored: Fri May 20 18:00:13 2016 -0700 Committer: Konstantin Boudnik <c...@apache.org> Committed: Wed May 25 10:57:16 2016 -0700 ---------------------------------------------------------------------- bigtop-tests/smoke-tests/README | 9 +++++++++ bigtop-tests/smoke-tests/spark/TestSpark.groovy | 18 ++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/6e81da1e/bigtop-tests/smoke-tests/README ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/README b/bigtop-tests/smoke-tests/README index 54d060d..759f53d 100644 --- a/bigtop-tests/smoke-tests/README +++ b/bigtop-tests/smoke-tests/README @@ -31,6 +31,15 @@ export TAJO_HOME="/usr/lib/tajo" export OOZIE_TAR_HOME="/usr/lib/oozie/doc" ``` +Fallback mode for shark-shell submission is 'yarn-client'. For a good reason, +we aren't using YARN to provision cluster services. Hence, in order to execute +the test against a standalone Spark cluster, like we deploy would require two +more environment variables: +` +export SPARK_MASTER_IP=bigtop1.docker +export SPARK_MASTER_PORT=7077 +` + Then, simply invoke the tests you want to run, as described below. Each directory is a gradle "subproject" mapping to an ecosystem component. http://git-wip-us.apache.org/repos/asf/bigtop/blob/6e81da1e/bigtop-tests/smoke-tests/spark/TestSpark.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/spark/TestSpark.groovy b/bigtop-tests/smoke-tests/spark/TestSpark.groovy index 4a715b3..8908c3d 100644 --- a/bigtop-tests/smoke-tests/spark/TestSpark.groovy +++ b/bigtop-tests/smoke-tests/spark/TestSpark.groovy @@ -40,7 +40,6 @@ class TestSpark { static Shell sh = new Shell("/bin/bash -s") static final String SPARK_HOME = System.getenv("SPARK_HOME") - static final String SPARK_SHELL = SPARK_HOME + "/bin/spark-shell --master yarn-client" static final String TEST_SPARKSQL_LOG = "/tmp/TestSpark_testSparkSQL.log" @BeforeClass @@ -69,7 +68,22 @@ class TestSpark { @Test void testSparkSQL() { - sh.exec(SPARK_SHELL + " --class org.apache.spark.examples.sql.JavaSparkSQL " + " --jars " + SPARK_HOME + "/lib/spark-examples*.jar > " + TEST_SPARKSQL_LOG + " 2>&1") + // Let's figure out the proper mode for the submission + // If SPARK_MASTER_IP nor SPARK_MASTER_PORT are set, we'll assume + // 'yarn-client' mode + String masterMode = 'yarn-client' + if (System.env.SPARK_MASTER_IP != null && System.env.SPARK_MASTER_PORT != null) + masterMode = "spark://$MASTER_IP:$MASTER_PORT" + else + println("SPARK_MASTER isn't set. yarn-client submission will be used. " + + "Refer to smoke-tests/README If this isn't what you you expect.") + + final String SPARK_SHELL = SPARK_HOME + "/bin/spark-shell --master $masterMode" + // Let's use time, 'cause the test has one job + sh.exec("timeout 120 " + SPARK_SHELL + + " --class org.apache.spark.examples.sql.JavaSparkSQL " + + " --jars " + SPARK_HOME + "/lib/spark-examples*.jar > " + + TEST_SPARKSQL_LOG + " 2>&1") logError(sh) assertTrue("Failed ...", sh.getRet() == 0); }