Repository: spark Updated Branches: refs/heads/master c44bf137c -> 46d98e0a1
[SPARK-16028][SPARKR] spark.lapply can work with active context ## What changes were proposed in this pull request? spark.lapply and setLogLevel ## How was this patch tested? unit test shivaram thunterdb Author: Felix Cheung <felixcheun...@hotmail.com> Closes #13752 from felixcheung/rlapply. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/46d98e0a Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/46d98e0a Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/46d98e0a Branch: refs/heads/master Commit: 46d98e0a1f40a4c6ae92253c5c498a3a924497fc Parents: c44bf13 Author: Felix Cheung <felixcheun...@hotmail.com> Authored: Mon Jun 20 12:08:42 2016 -0700 Committer: Shivaram Venkataraman <shiva...@cs.berkeley.edu> Committed: Mon Jun 20 12:08:42 2016 -0700 ---------------------------------------------------------------------- R/pkg/R/context.R | 20 +++++++++++++------- R/pkg/inst/tests/testthat/test_context.R | 6 +++--- 2 files changed, 16 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/46d98e0a/R/pkg/R/context.R ---------------------------------------------------------------------- diff --git a/R/pkg/R/context.R b/R/pkg/R/context.R index 5c88603..968a9d2 100644 --- a/R/pkg/R/context.R +++ b/R/pkg/R/context.R @@ -252,17 +252,20 @@ setCheckpointDir <- function(sc, dirName) { #' } #' #' @rdname spark.lapply -#' @param sc Spark Context to use #' @param list the list of elements #' @param func a function that takes one argument. #' @return a list of results (the exact type being determined by the function) #' @export #' @examples #'\dontrun{ -#' sc <- sparkR.init() -#' doubled <- spark.lapply(sc, 1:10, function(x){2 * x}) +#' sparkR.session() +#' doubled <- spark.lapply(1:10, function(x){2 * x}) #'} -spark.lapply <- function(sc, list, func) { +spark.lapply <- function(list, func) { + if (!exists(".sparkRjsc", envir = .sparkREnv)) { + stop("SparkR has not been initialized. Please call sparkR.session()") + } + sc <- get(".sparkRjsc", envir = .sparkREnv) rdd <- parallelize(sc, list, length(list)) results <- map(rdd, func) local <- collect(results) @@ -274,14 +277,17 @@ spark.lapply <- function(sc, list, func) { #' Set new log level: "ALL", "DEBUG", "ERROR", "FATAL", "INFO", "OFF", "TRACE", "WARN" #' #' @rdname setLogLevel -#' @param sc Spark Context to use #' @param level New log level #' @export #' @examples #'\dontrun{ -#' setLogLevel(sc, "ERROR") +#' setLogLevel("ERROR") #'} -setLogLevel <- function(sc, level) { +setLogLevel <- function(level) { + if (!exists(".sparkRjsc", envir = .sparkREnv)) { + stop("SparkR has not been initialized. Please call sparkR.session()") + } + sc <- get(".sparkRjsc", envir = .sparkREnv) callJMethod(sc, "setLogLevel", level) } http://git-wip-us.apache.org/repos/asf/spark/blob/46d98e0a/R/pkg/inst/tests/testthat/test_context.R ---------------------------------------------------------------------- diff --git a/R/pkg/inst/tests/testthat/test_context.R b/R/pkg/inst/tests/testthat/test_context.R index f123187..b149818 100644 --- a/R/pkg/inst/tests/testthat/test_context.R +++ b/R/pkg/inst/tests/testthat/test_context.R @@ -107,8 +107,8 @@ test_that("job group functions can be called", { }) test_that("utility function can be called", { - sc <- sparkR.sparkContext() - setLogLevel(sc, "ERROR") + sparkR.sparkContext() + setLogLevel("ERROR") sparkR.session.stop() }) @@ -161,7 +161,7 @@ test_that("sparkJars sparkPackages as comma-separated strings", { test_that("spark.lapply should perform simple transforms", { sc <- sparkR.sparkContext() - doubled <- spark.lapply(sc, 1:10, function(x) { 2 * x }) + doubled <- spark.lapply(1:10, function(x) { 2 * x }) expect_equal(doubled, as.list(2 * 1:10)) sparkR.session.stop() }) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
