Github user felixcheung commented on a diff in the pull request:
https://github.com/apache/spark/pull/22954#discussion_r232499902
--- Diff: R/pkg/R/SQLContext.R ---
@@ -147,6 +147,91 @@ getDefaultSqlSource <- function() {
l[["spark.sql.sources.default"]]
}
+writeToTempFileInArrow <- function(rdf, numPartitions) {
+ requireNamespace1 <- requireNamespace
+
+ # For some reasons, Arrow R API requires to load 'defer_parent' which is
from 'withr' package.
+ # This is a workaround to avoid this error. Otherwise, we should
directly load 'withr'
+ # package, which CRAN complains about.
+ defer_parent <- function(x, ...) {
+ if (requireNamespace1("withr", quietly = TRUE)) {
+ defer_parent <- get("defer_parent", envir = asNamespace("withr"),
inherits = FALSE)
+ defer_parent(x, ...)
+ } else {
+ stop("'withr' package should be installed.")
+ }
+ }
+
+ # R API in Arrow is not yet released. CRAN requires to add the package
in requireNamespace
+ # at DESCRIPTION. Later, CRAN checks if the package is available or not.
Therefore, it works
+ # around by avoiding direct requireNamespace.
+ if (requireNamespace1("arrow", quietly = TRUE)) {
+ record_batch <- get("record_batch", envir = asNamespace("arrow"),
inherits = FALSE)
+ record_batch_stream_writer <- get(
+ "record_batch_stream_writer", envir = asNamespace("arrow"), inherits
= FALSE)
+ file_output_stream <- get(
+ "file_output_stream", envir = asNamespace("arrow"), inherits = FALSE)
+ write_record_batch <- get(
+ "write_record_batch", envir = asNamespace("arrow"), inherits = FALSE)
+
+ numPartitions <- if (!is.null(numPartitions)) {
+ numToInt(numPartitions)
+ } else {
+ 1
--- End diff --
future: consolidate the default here and inside makeSplits
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
