Repository: spark Updated Branches: refs/heads/master 89ea0041a -> f21e2da03
[SPARK-10807] [SPARKR] Added as.data.frame as a synonym for collect Created method as.data.frame as a synonym for collect(). Author: Oscar D. Lara Yejas <[email protected]> Author: olarayej <[email protected]> Author: Oscar D. Lara Yejas <[email protected]> Closes #8908 from olarayej/SPARK-10807. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f21e2da0 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f21e2da0 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f21e2da0 Branch: refs/heads/master Commit: f21e2da03fbf8041fece476e3d5c699aef819451 Parents: 89ea004 Author: Oscar D. Lara Yejas <[email protected]> Authored: Wed Sep 30 18:03:31 2015 -0700 Committer: Shivaram Venkataraman <[email protected]> Committed: Wed Sep 30 18:03:31 2015 -0700 ---------------------------------------------------------------------- R/pkg/NAMESPACE | 2 ++ R/pkg/R/DataFrame.R | 25 +++++++++++++++++++++++++ R/pkg/R/generics.R | 4 ++++ R/pkg/inst/tests/test_sparkSQL.R | 9 ++++++++- 4 files changed, 39 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/f21e2da0/R/pkg/NAMESPACE ---------------------------------------------------------------------- diff --git a/R/pkg/NAMESPACE b/R/pkg/NAMESPACE index 9d39630..c28c47d 100644 --- a/R/pkg/NAMESPACE +++ b/R/pkg/NAMESPACE @@ -247,3 +247,5 @@ export("structField", "structType.jobj", "structType.structField", "print.structType") + +export("as.data.frame") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/f21e2da0/R/pkg/R/DataFrame.R ---------------------------------------------------------------------- diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R index c3c1893..65e368c 100644 --- a/R/pkg/R/DataFrame.R +++ b/R/pkg/R/DataFrame.R @@ -1848,3 +1848,28 @@ setMethod("crosstab", sct <- callJMethod(statFunctions, "crosstab", col1, col2) collect(dataFrame(sct)) }) + + +#' This function downloads the contents of a DataFrame into an R's data.frame. +#' Since data.frames are held in memory, ensure that you have enough memory +#' in your system to accommodate the contents. +#' +#' @title Download data from a DataFrame into a data.frame +#' @param x a DataFrame +#' @return a data.frame +#' @rdname as.data.frame +#' @examples \dontrun{ +#' +#' irisDF <- createDataFrame(sqlContext, iris) +#' df <- as.data.frame(irisDF[irisDF$Species == "setosa", ]) +#' } +setMethod("as.data.frame", + signature(x = "DataFrame"), + function(x, ...) { + # Check if additional parameters have been passed + if (length(list(...)) > 0) { + stop(paste("Unused argument(s): ", paste(list(...), collapse=", "))) + } + collect(x) + } +) http://git-wip-us.apache.org/repos/asf/spark/blob/f21e2da0/R/pkg/R/generics.R ---------------------------------------------------------------------- diff --git a/R/pkg/R/generics.R b/R/pkg/R/generics.R index 43dd8d2..3db41e0 100644 --- a/R/pkg/R/generics.R +++ b/R/pkg/R/generics.R @@ -983,3 +983,7 @@ setGeneric("glm") #' @rdname rbind #' @export setGeneric("rbind", signature = "...") + +#' @rdname as.data.frame +#' @export +setGeneric("as.data.frame") \ No newline at end of file http://git-wip-us.apache.org/repos/asf/spark/blob/f21e2da0/R/pkg/inst/tests/test_sparkSQL.R ---------------------------------------------------------------------- diff --git a/R/pkg/inst/tests/test_sparkSQL.R b/R/pkg/inst/tests/test_sparkSQL.R index e159a69..8f85eec 100644 --- a/R/pkg/inst/tests/test_sparkSQL.R +++ b/R/pkg/inst/tests/test_sparkSQL.R @@ -1327,6 +1327,13 @@ test_that("SQL error message is returned from JVM", { expect_equal(grepl("Table Not Found: blah", retError), TRUE) }) +test_that("Method as.data.frame as a synonym for collect()", { + irisDF <- createDataFrame(sqlContext, iris) + expect_equal(as.data.frame(irisDF), collect(irisDF)) + irisDF2 <- irisDF[irisDF$Species == "setosa", ] + expect_equal(as.data.frame(irisDF2), collect(irisDF2)) +}) + unlink(parquetPath) unlink(jsonPath) -unlink(jsonPathNa) +unlink(jsonPathNa) \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
