Repository: spark Updated Branches: refs/heads/branch-2.2 755152482 -> 0568f289d
[SPARK-22344][SPARKR] clean up install dir if running test as source package ## What changes were proposed in this pull request? remove spark if spark downloaded & installed ## How was this patch tested? manually by building package Jenkins, AppVeyor Author: Felix Cheung <felixcheun...@hotmail.com> Closes #19657 from felixcheung/rinstalldir. (cherry picked from commit b70aa9e08b4476746e912c2c2a8b7bdd102305e8) Signed-off-by: Felix Cheung <felixche...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0568f289 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0568f289 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0568f289 Branch: refs/heads/branch-2.2 Commit: 0568f289d92a32ee09290d6591d8aa2bb4b72531 Parents: 7551524 Author: Felix Cheung <felixcheun...@hotmail.com> Authored: Fri Nov 10 10:22:42 2017 -0800 Committer: Felix Cheung <felixche...@apache.org> Committed: Fri Nov 10 10:22:56 2017 -0800 ---------------------------------------------------------------------- R/pkg/R/install.R | 37 ++++++++++++++++++++++++++++++- R/pkg/R/utils.R | 13 +++++++++++ R/pkg/tests/fulltests/test_utils.R | 25 +++++++++++++++++++++ R/pkg/tests/run-all.R | 4 +++- R/pkg/vignettes/sparkr-vignettes.Rmd | 6 ++++- 5 files changed, 82 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/0568f289/R/pkg/R/install.R ---------------------------------------------------------------------- diff --git a/R/pkg/R/install.R b/R/pkg/R/install.R index 492dee6..04dc756 100644 --- a/R/pkg/R/install.R +++ b/R/pkg/R/install.R @@ -152,6 +152,11 @@ install.spark <- function(hadoopVersion = "2.7", mirrorUrl = NULL, }) if (!tarExists || overwrite || !success) { unlink(packageLocalPath) + if (success) { + # if tar file was not there before (or it was, but we are told to overwrite it), + # and untar is successful - set a flag that we have downloaded (and untar) Spark package. + assign(".sparkDownloaded", TRUE, envir = .sparkREnv) + } } if (!success) stop("Extract archive failed.") message("DONE.") @@ -266,6 +271,7 @@ hadoopVersionName <- function(hadoopVersion) { # The implementation refers to appdirs package: https://pypi.python.org/pypi/appdirs and # adapt to Spark context +# see also sparkCacheRelPathLength() sparkCachePath <- function() { if (is_windows()) { winAppPath <- Sys.getenv("LOCALAPPDATA", unset = NA) @@ -282,7 +288,7 @@ sparkCachePath <- function() { } } else if (.Platform$OS.type == "unix") { if (Sys.info()["sysname"] == "Darwin") { - path <- file.path(Sys.getenv("HOME"), "Library/Caches", "spark") + path <- file.path(Sys.getenv("HOME"), "Library", "Caches", "spark") } else { path <- file.path( Sys.getenv("XDG_CACHE_HOME", file.path(Sys.getenv("HOME"), ".cache")), "spark") @@ -293,6 +299,16 @@ sparkCachePath <- function() { normalizePath(path, mustWork = FALSE) } +# Length of the Spark cache specific relative path segments for each platform +# eg. "Apache\Spark\Cache" is 3 in Windows, or "spark" is 1 in unix +# Must match sparkCachePath() exactly. +sparkCacheRelPathLength <- function() { + if (is_windows()) { + 3 + } else { + 1 + } +} installInstruction <- function(mode) { if (mode == "remote") { @@ -310,3 +326,22 @@ installInstruction <- function(mode) { stop(paste0("No instruction found for ", mode, " mode.")) } } + +uninstallDownloadedSpark <- function() { + # clean up if Spark was downloaded + sparkDownloaded <- getOne(".sparkDownloaded", + envir = .sparkREnv, + inherits = TRUE, + ifnotfound = FALSE) + sparkDownloadedDir <- Sys.getenv("SPARK_HOME") + if (sparkDownloaded && nchar(sparkDownloadedDir) > 0) { + unlink(sparkDownloadedDir, recursive = TRUE, force = TRUE) + + dirs <- traverseParentDirs(sparkCachePath(), sparkCacheRelPathLength()) + lapply(dirs, function(d) { + if (length(list.files(d, all.files = TRUE, include.dirs = TRUE, no.. = TRUE)) == 0) { + unlink(d, recursive = TRUE, force = TRUE) + } + }) + } +} http://git-wip-us.apache.org/repos/asf/spark/blob/0568f289/R/pkg/R/utils.R ---------------------------------------------------------------------- diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R index 7225da9..da5f3cf 100644 --- a/R/pkg/R/utils.R +++ b/R/pkg/R/utils.R @@ -911,3 +911,16 @@ hadoop_home_set <- function() { windows_with_hadoop <- function() { !is_windows() || hadoop_home_set() } + +# get0 not supported before R 3.2.0 +getOne <- function(x, envir, inherits = TRUE, ifnotfound = NULL) { + mget(x[1L], envir = envir, inherits = inherits, ifnotfound = list(ifnotfound))[[1L]] +} + +# Returns a vector of parent directories, traversing up count times, starting with a full path +# eg. traverseParentDirs("/Users/user/Library/Caches/spark/spark2.2", 1) should return +# this "/Users/user/Library/Caches/spark/spark2.2" +# and "/Users/user/Library/Caches/spark" +traverseParentDirs <- function(x, count) { + if (dirname(x) == x || count <= 0) x else c(x, Recall(dirname(x), count - 1)) +} http://git-wip-us.apache.org/repos/asf/spark/blob/0568f289/R/pkg/tests/fulltests/test_utils.R ---------------------------------------------------------------------- diff --git a/R/pkg/tests/fulltests/test_utils.R b/R/pkg/tests/fulltests/test_utils.R index 8cfdc95..50fc6f3 100644 --- a/R/pkg/tests/fulltests/test_utils.R +++ b/R/pkg/tests/fulltests/test_utils.R @@ -236,4 +236,29 @@ test_that("basenameSansExtFromUrl", { expect_equal(basenameSansExtFromUrl(z), "spark-2.1.0--hive") }) +test_that("getOne", { + dummy <- getOne(".dummyValue", envir = new.env(), ifnotfound = FALSE) + expect_equal(dummy, FALSE) +}) + +test_that("traverseParentDirs", { + if (is_windows()) { + # original path is included as-is, otherwise dirname() replaces \\ with / on windows + dirs <- traverseParentDirs("c:\\Users\\user\\AppData\\Local\\Apache\\Spark\\Cache\\spark2.2", 3) + expect <- c("c:\\Users\\user\\AppData\\Local\\Apache\\Spark\\Cache\\spark2.2", + "c:/Users/user/AppData/Local/Apache/Spark/Cache", + "c:/Users/user/AppData/Local/Apache/Spark", + "c:/Users/user/AppData/Local/Apache") + expect_equal(dirs, expect) + } else { + dirs <- traverseParentDirs("/Users/user/Library/Caches/spark/spark2.2", 1) + expect <- c("/Users/user/Library/Caches/spark/spark2.2", "/Users/user/Library/Caches/spark") + expect_equal(dirs, expect) + + dirs <- traverseParentDirs("/home/u/.cache/spark/spark2.2", 1) + expect <- c("/home/u/.cache/spark/spark2.2", "/home/u/.cache/spark") + expect_equal(dirs, expect) + } +}) + sparkR.session.stop() http://git-wip-us.apache.org/repos/asf/spark/blob/0568f289/R/pkg/tests/run-all.R ---------------------------------------------------------------------- diff --git a/R/pkg/tests/run-all.R b/R/pkg/tests/run-all.R index 3f432f7..4a897d5 100644 --- a/R/pkg/tests/run-all.R +++ b/R/pkg/tests/run-all.R @@ -46,7 +46,7 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) { tmpDir <- tempdir() tmpArg <- paste0("-Djava.io.tmpdir=", tmpDir) sparkRTestConfig <- list(spark.driver.extraJavaOptions = tmpArg, - spark.executor.extraJavaOptions = tmpArg) + spark.executor.extraJavaOptions = tmpArg) } test_package("SparkR") @@ -58,3 +58,5 @@ if (identical(Sys.getenv("NOT_CRAN"), "true")) { NULL, "summary") } + +SparkR:::uninstallDownloadedSpark() http://git-wip-us.apache.org/repos/asf/spark/blob/0568f289/R/pkg/vignettes/sparkr-vignettes.Rmd ---------------------------------------------------------------------- diff --git a/R/pkg/vignettes/sparkr-vignettes.Rmd b/R/pkg/vignettes/sparkr-vignettes.Rmd index 240dda3..ad660ec 100644 --- a/R/pkg/vignettes/sparkr-vignettes.Rmd +++ b/R/pkg/vignettes/sparkr-vignettes.Rmd @@ -37,7 +37,7 @@ opts_hooks$set(eval = function(options) { options }) r_tmp_dir <- tempdir() -tmp_arg <- paste("-Djava.io.tmpdir=", r_tmp_dir, sep = "") +tmp_arg <- paste0("-Djava.io.tmpdir=", r_tmp_dir) sparkSessionConfig <- list(spark.driver.extraJavaOptions = tmp_arg, spark.executor.extraJavaOptions = tmp_arg) old_java_opt <- Sys.getenv("_JAVA_OPTIONS") @@ -1148,3 +1148,7 @@ env | map ```{r, echo=FALSE} sparkR.session.stop() ``` + +```{r cleanup, include=FALSE} +SparkR:::uninstallDownloadedSpark() +``` --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org