[bigtop] branch master updated: BIGTOP-3372. Building Spark fails on Debian 9 and Ubuntu 16.04. (#648)

[sekikn]

This is an automated email from the ASF dual-hosted git repository.

sekikn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git


The following commit(s) were added to refs/heads/master by this push:
     new 0a7bc75  BIGTOP-3372. Building Spark fails on Debian 9 and Ubuntu 
16.04. (#648)
0a7bc75 is described below

commit 0a7bc759de1917cbf8661438d6f8535298be6454
Author: Kengo Seki <sek...@apache.org>
AuthorDate: Fri Jul 3 20:52:37 2020 +0900

    BIGTOP-3372. Building Spark fails on Debian 9 and Ubuntu 16.04. (#648)
---
 .../src/common/spark/patch0-SPARK-24152.diff       | 13 ++++++++++++
 bigtop-packages/src/rpm/spark/SPECS/spark.spec     |  3 +++
 bigtop-tests/smoke-tests/spark/TestSpark.groovy    | 23 ++++++++++++++++++++++
 3 files changed, 39 insertions(+)

diff --git a/bigtop-packages/src/common/spark/patch0-SPARK-24152.diff 
b/bigtop-packages/src/common/spark/patch0-SPARK-24152.diff
new file mode 100644
index 0000000..ce343f9
--- /dev/null
+++ b/bigtop-packages/src/common/spark/patch0-SPARK-24152.diff
@@ -0,0 +1,13 @@
+diff --git a/R/check-cran.sh b/R/check-cran.sh
+index 22cc9c6b60..3d83ce3a85 100755
+--- a/R/check-cran.sh
++++ b/R/check-cran.sh
+@@ -51,7 +51,7 @@ fi
+ # Run check as-cran.
+ VERSION=`grep Version "$FWDIR/pkg/DESCRIPTION" | awk '{print $NF}'`
+ 
+-CRAN_CHECK_OPTIONS="--as-cran"
++CRAN_CHECK_OPTIONS=""
+ 
+ if [ -n "$NO_TESTS" ]
+ then
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec 
b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index 9e43769..560af1d 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -56,6 +56,7 @@ Source6: init.d.tmpl
 Source7: spark-history-server.svc
 Source8: spark-thriftserver.svc
 Source9: bigtop.bom
+#BIGTOP_PATCH_FILES
 Requires: bigtop-utils >= 0.7, hadoop-client, hadoop-yarn
 Requires(preun): /sbin/service
 
@@ -157,6 +158,8 @@ SparkR is an R package that provides a light-weight 
frontend to use Apache Spark
 %prep
 %setup -n %{spark_name}-%{spark_base_version}
 
+#BIGTOP_PATCH_COMMANDS
+
 %build
 bash $RPM_SOURCE_DIR/do-component-build
 
diff --git a/bigtop-tests/smoke-tests/spark/TestSpark.groovy 
b/bigtop-tests/smoke-tests/spark/TestSpark.groovy
index 6d13ebf..a0e7a40 100644
--- a/bigtop-tests/smoke-tests/spark/TestSpark.groovy
+++ b/bigtop-tests/smoke-tests/spark/TestSpark.groovy
@@ -43,6 +43,7 @@ class TestSpark {
   static final String SPARK_MASTER_IP = System.getenv("SPARK_MASTER_IP")
   static final String SPARK_MASTER_PORT = System.getenv("SPARK_MASTER_PORT")
   static final String TEST_SPARKSQL_LOG = "/tmp/TestSpark_testSparkSQL.log"
+  static final String TEST_SPARKR_LOG = "/tmp/TestSpark_testSparkR.log"
 
   @BeforeClass
   static void setUp() {
@@ -89,4 +90,26 @@ class TestSpark {
     logError(sh)
     assertTrue("Failed ...", sh.getRet() == 0);
   }
+
+  @Test
+  void testSparkR() {
+    String masterMode = 'yarn-client'
+    if (SPARK_MASTER_IP != null && SPARK_MASTER_PORT != null)
+      masterMode = "spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT"
+    else
+      println("SPARK_MASTER isn't set. yarn-client submission will be used. " +
+          "Refer to smoke-tests/README If this isn't what you you expect.")
+
+    new File('/tmp/dataframe.R').withWriter { writer ->
+      new File(SPARK_HOME + "/examples/src/main/r/dataframe.R").eachLine { 
line ->
+        writer << line.replace('file.path(Sys.getenv("SPARK_HOME"), ', 
'file.path(') +
+          System.getProperty("line.separator")
+      }
+    }
+
+    final String SPARK_SUBMIT = SPARK_HOME + "/bin/spark-submit --master 
$masterMode"
+    sh.exec("timeout 120 " + SPARK_SUBMIT + " /tmp/dataframe.R > " + 
TEST_SPARKR_LOG + " 2>&1")
+    logError(sh)
+    assertTrue("Failed to execute SparkR script", sh.getRet() == 0);
+  }
 }