spark git commit: [SPARKR] Require Java 8 for SparkR

2018-05-11 Thread shivaram 
Repository: spark
Updated Branches:
  refs/heads/branch-2.3 1d598b771 -> 7de4bef9e


[SPARKR] Require Java 8 for SparkR

This change updates the SystemRequirements and also includes a runtime check if 
the JVM is being launched by R. The runtime check is done by querying `java 
-version`

## How was this patch tested?

Tested on a Mac and Windows machine

Author: Shivaram Venkataraman 

Closes #21278 from shivaram/sparkr-skip-solaris.

(cherry picked from commit f27a035daf705766d3445e5c6a99867c11c552b0)
Signed-off-by: Shivaram Venkataraman 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7de4bef9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7de4bef9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7de4bef9

Branch: refs/heads/branch-2.3
Commit: 7de4bef9ec37440aa36e6b0e9d8656de07d03b68
Parents: 1d598b7
Author: Shivaram Venkataraman 
Authored: Fri May 11 17:00:51 2018 -0700
Committer: Shivaram Venkataraman 
Committed: Fri May 11 17:01:02 2018 -0700

--
 R/pkg/DESCRIPTION |  1 +
 R/pkg/R/client.R  | 35 +++
 R/pkg/R/sparkR.R  |  1 +
 R/pkg/R/utils.R   |  4 ++--
 4 files changed, 39 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/7de4bef9/R/pkg/DESCRIPTION
--
diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 29a8a00..632bcb3 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -13,6 +13,7 @@ Authors@R: c(person("Shivaram", "Venkataraman", role = 
c("aut", "cre"),
 License: Apache License (== 2.0)
 URL: http://www.apache.org/ http://spark.apache.org/
 BugReports: http://spark.apache.org/contributing.html
+SystemRequirements: Java (== 8)
 Depends:
 R (>= 3.0),
 methods

http://git-wip-us.apache.org/repos/asf/spark/blob/7de4bef9/R/pkg/R/client.R
--
diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R
index 7244cc9..e9295e0 100644
--- a/R/pkg/R/client.R
+++ b/R/pkg/R/client.R
@@ -60,6 +60,40 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, 
sparkSubmitOpts, pack
   combinedArgs
 }
 
+checkJavaVersion <- function() {
+  javaBin <- "java"
+  javaHome <- Sys.getenv("JAVA_HOME")
+  javaReqs <- utils::packageDescription(utils::packageName(), 
fields=c("SystemRequirements"))
+  sparkJavaVersion <- as.numeric(tail(strsplit(javaReqs, "[(=)]")[[1]], n = 
1L))
+  if (javaHome != "") {
+javaBin <- file.path(javaHome, "bin", javaBin)
+  }
+
+  # If java is missing from PATH, we get an error in Unix and a warning in 
Windows
+  javaVersionOut <- tryCatch(
+  launchScript(javaBin, "-version", wait = TRUE, stdout = TRUE, stderr = 
TRUE),
+   error = function(e) {
+ stop("Java version check failed. Please make sure Java is 
installed",
+  " and set JAVA_HOME to point to the installation 
directory.", e)
+   },
+   warning = function(w) {
+ stop("Java version check failed. Please make sure Java is 
installed",
+  " and set JAVA_HOME to point to the installation 
directory.", w)
+   })
+  javaVersionFilter <- Filter(
+  function(x) {
+grepl("java version", x)
+  }, javaVersionOut)
+
+  javaVersionStr <- strsplit(javaVersionFilter[[1]], "[\"]")[[1L]][2]
+  # javaVersionStr is of the form 1.8.0_92.
+  # Extract 8 from it to compare to sparkJavaVersion
+  javaVersionNum <- as.integer(strsplit(javaVersionStr, "[.]")[[1L]][2])
+  if (javaVersionNum != sparkJavaVersion) {
+stop(paste("Java version", sparkJavaVersion, "is required for this 
package; found version:", javaVersionStr))
+  }
+}
+
 launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) {
   sparkSubmitBinName <- determineSparkSubmitBin()
   if (sparkHome != "") {
@@ -67,6 +101,7 @@ launchBackend <- function(args, sparkHome, jars, 
sparkSubmitOpts, packages) {
   } else {
 sparkSubmitBin <- sparkSubmitBinName
   }
+
   combinedArgs <- generateSparkSubmitArgs(args, sparkHome, jars, 
sparkSubmitOpts, packages)
   cat("Launching java with spark-submit command", sparkSubmitBin, 
combinedArgs, "\n")
   invisible(launchScript(sparkSubmitBin, combinedArgs))

http://git-wip-us.apache.org/repos/asf/spark/blob/7de4bef9/R/pkg/R/sparkR.R
--
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
index 7430d84..2cd8b0c 100644
--- a/R/pkg/R/sparkR.R
+++ b/R/pkg/R/sparkR.R
@@ -170,6 +170,7 @@ sparkR.sparkContext <- function(
 submitOps <- getClientModeSparkSubmitOpts(

spark git commit: [SPARKR] Require Java 8 for SparkR

2018-05-11 Thread shivaram 
Repository: spark
Updated Branches:
  refs/heads/master 92f6f52ff -> f27a035da


[SPARKR] Require Java 8 for SparkR

This change updates the SystemRequirements and also includes a runtime check if 
the JVM is being launched by R. The runtime check is done by querying `java 
-version`

## How was this patch tested?

Tested on a Mac and Windows machine

Author: Shivaram Venkataraman 

Closes #21278 from shivaram/sparkr-skip-solaris.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f27a035d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f27a035d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f27a035d

Branch: refs/heads/master
Commit: f27a035daf705766d3445e5c6a99867c11c552b0
Parents: 92f6f52
Author: Shivaram Venkataraman 
Authored: Fri May 11 17:00:51 2018 -0700
Committer: Shivaram Venkataraman 
Committed: Fri May 11 17:00:51 2018 -0700

--
 R/pkg/DESCRIPTION |  1 +
 R/pkg/R/client.R  | 35 +++
 R/pkg/R/sparkR.R  |  1 +
 R/pkg/R/utils.R   |  4 ++--
 4 files changed, 39 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/f27a035d/R/pkg/DESCRIPTION
--
diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 855eb5b..f52d785 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -13,6 +13,7 @@ Authors@R: c(person("Shivaram", "Venkataraman", role = 
c("aut", "cre"),
 License: Apache License (== 2.0)
 URL: http://www.apache.org/ http://spark.apache.org/
 BugReports: http://spark.apache.org/contributing.html
+SystemRequirements: Java (== 8)
 Depends:
 R (>= 3.0),
 methods

http://git-wip-us.apache.org/repos/asf/spark/blob/f27a035d/R/pkg/R/client.R
--
diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R
index 7244cc9..e9295e0 100644
--- a/R/pkg/R/client.R
+++ b/R/pkg/R/client.R
@@ -60,6 +60,40 @@ generateSparkSubmitArgs <- function(args, sparkHome, jars, 
sparkSubmitOpts, pack
   combinedArgs
 }
 
+checkJavaVersion <- function() {
+  javaBin <- "java"
+  javaHome <- Sys.getenv("JAVA_HOME")
+  javaReqs <- utils::packageDescription(utils::packageName(), 
fields=c("SystemRequirements"))
+  sparkJavaVersion <- as.numeric(tail(strsplit(javaReqs, "[(=)]")[[1]], n = 
1L))
+  if (javaHome != "") {
+javaBin <- file.path(javaHome, "bin", javaBin)
+  }
+
+  # If java is missing from PATH, we get an error in Unix and a warning in 
Windows
+  javaVersionOut <- tryCatch(
+  launchScript(javaBin, "-version", wait = TRUE, stdout = TRUE, stderr = 
TRUE),
+   error = function(e) {
+ stop("Java version check failed. Please make sure Java is 
installed",
+  " and set JAVA_HOME to point to the installation 
directory.", e)
+   },
+   warning = function(w) {
+ stop("Java version check failed. Please make sure Java is 
installed",
+  " and set JAVA_HOME to point to the installation 
directory.", w)
+   })
+  javaVersionFilter <- Filter(
+  function(x) {
+grepl("java version", x)
+  }, javaVersionOut)
+
+  javaVersionStr <- strsplit(javaVersionFilter[[1]], "[\"]")[[1L]][2]
+  # javaVersionStr is of the form 1.8.0_92.
+  # Extract 8 from it to compare to sparkJavaVersion
+  javaVersionNum <- as.integer(strsplit(javaVersionStr, "[.]")[[1L]][2])
+  if (javaVersionNum != sparkJavaVersion) {
+stop(paste("Java version", sparkJavaVersion, "is required for this 
package; found version:", javaVersionStr))
+  }
+}
+
 launchBackend <- function(args, sparkHome, jars, sparkSubmitOpts, packages) {
   sparkSubmitBinName <- determineSparkSubmitBin()
   if (sparkHome != "") {
@@ -67,6 +101,7 @@ launchBackend <- function(args, sparkHome, jars, 
sparkSubmitOpts, packages) {
   } else {
 sparkSubmitBin <- sparkSubmitBinName
   }
+
   combinedArgs <- generateSparkSubmitArgs(args, sparkHome, jars, 
sparkSubmitOpts, packages)
   cat("Launching java with spark-submit command", sparkSubmitBin, 
combinedArgs, "\n")
   invisible(launchScript(sparkSubmitBin, combinedArgs))

http://git-wip-us.apache.org/repos/asf/spark/blob/f27a035d/R/pkg/R/sparkR.R
--
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
index 38ee794..d6a2d08 100644
--- a/R/pkg/R/sparkR.R
+++ b/R/pkg/R/sparkR.R
@@ -167,6 +167,7 @@ sparkR.sparkContext <- function(
 submitOps <- getClientModeSparkSubmitOpts(
 Sys.getenv("SPARKR_SUBMIT_ARGS", "sparkr-shell"),
 sparkEnvirMap)
+checkJavaVersion()
 launchBackend(
 args =