This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 02b4be5 BIGTOP-3570. Fix Spark smoke test failures. (#798)
02b4be5 is described below
commit 02b4be54ae3dd13dadc7d55a7aa704209bd1aa97
Author: Kengo Seki <[email protected]>
AuthorDate: Tue Jul 13 10:23:16 2021 +0900
BIGTOP-3570. Fix Spark smoke test failures. (#798)
---
bigtop-tests/smoke-tests/README | 2 +-
bigtop-tests/smoke-tests/spark/TestSpark.groovy | 22 +++++++++++-----------
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/bigtop-tests/smoke-tests/README b/bigtop-tests/smoke-tests/README
index b61871f..5e8bd6a 100644
--- a/bigtop-tests/smoke-tests/README
+++ b/bigtop-tests/smoke-tests/README
@@ -27,7 +27,7 @@ export SPARK_HOME="/usr/lib/spark"
export OOZIE_TAR_HOME="/usr/lib/oozie/doc"
```
-Fallback mode for shark-shell submission is 'yarn-client'. For a good reason,
+Fallback master URL for shark-shell submission is 'yarn'. For a good reason,
we aren't using YARN to provision cluster services. Hence, in order to execute
the test against a standalone Spark cluster, like we deploy would require two
more environment variables:
diff --git a/bigtop-tests/smoke-tests/spark/TestSpark.groovy
b/bigtop-tests/smoke-tests/spark/TestSpark.groovy
index c2327ce..82bfe0d 100644
--- a/bigtop-tests/smoke-tests/spark/TestSpark.groovy
+++ b/bigtop-tests/smoke-tests/spark/TestSpark.groovy
@@ -72,19 +72,19 @@ class TestSpark {
@Test
void testSparkSQL() {
// Let's figure out the proper mode for the submission
- // If SPARK_MASTER_IP nor SPARK_MASTER_PORT are set, we'll assume
- // 'yarn-client' mode
- String masterMode = 'yarn-client'
+ // If SPARK_MASTER_IP nor SPARK_MASTER_PORT are not set,
+ // we'll assume 'yarn' as the master URL.
+ String master = 'yarn'
if (SPARK_MASTER_IP != null && SPARK_MASTER_PORT != null)
- masterMode = "spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT"
+ master = "spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT"
else
- println("SPARK_MASTER isn't set. yarn-client submission will be used. " +
+ println("SPARK_MASTER isn't set. yarn will be used as the master URL. " +
"Refer to smoke-tests/README If this isn't what you you expect.")
- final String SPARK_SHELL = SPARK_HOME + "/bin/spark-shell --master
$masterMode"
+ final String SPARK_SHELL = SPARK_HOME + "/bin/spark-shell --master $master"
// Let's use time, 'cause the test has one job
sh.exec("timeout 300 " + SPARK_SHELL +
- " --class org.apache.spark.examples.sql.JavaSparkSQLExample " +
+ " --class org.apache.spark.examples.sql.SparkSQLExample " +
" --jars " + SPARK_HOME + "/examples/jars/spark-examples*.jar > " +
TEST_SPARKSQL_LOG + " 2>&1")
logError(sh)
@@ -93,11 +93,11 @@ class TestSpark {
@Test
void testSparkR() {
- String masterMode = 'yarn-client'
+ String master = 'yarn'
if (SPARK_MASTER_IP != null && SPARK_MASTER_PORT != null)
- masterMode = "spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT"
+ master = "spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT"
else
- println("SPARK_MASTER isn't set. yarn-client submission will be used. " +
+ println("SPARK_MASTER isn't set. yarn will be used as the master URL. " +
"Refer to smoke-tests/README If this isn't what you you expect.")
new File('/tmp/dataframe.R').withWriter { writer ->
@@ -107,7 +107,7 @@ class TestSpark {
}
}
- final String SPARK_SUBMIT = SPARK_HOME + "/bin/spark-submit --master
$masterMode"
+ final String SPARK_SUBMIT = SPARK_HOME + "/bin/spark-submit --master
$master"
sh.exec("timeout 300 " + SPARK_SUBMIT + " /tmp/dataframe.R > " +
TEST_SPARKR_LOG + " 2>&1")
logError(sh)
assertTrue("Failed to execute SparkR script", sh.getRet() == 0);
