felixcheung commented on a change in pull request #23969: [SPARK-26920][R]
Deduplicate type checking across Arrow optimization in SparkR
URL: https://github.com/apache/spark/pull/23969#discussion_r263989032
##########
File path: R/pkg/R/SQLContext.R
##########
@@ -197,17 +197,40 @@ writeToFileInArrow <- function(fileName, rdf,
numPartitions) {
}
}
-checkTypeRequirementForArrow <- function(dataHead, schema) {
- # Currenty Arrow optimization does not support raw for now.
- # Also, it does not support explicit float type set by users. It leads to
- # incorrect conversion. We will fall back to the path without Arrow
optimization.
- if (any(sapply(dataHead, is.raw))) {
- stop("Arrow optimization with R DataFrame does not support raw type yet.")
- }
- if (inherits(schema, "structType")) {
- if (any(sapply(schema$fields(), function(x) x$dataType.toString() ==
"FloatType"))) {
- stop("Arrow optimization with R DataFrame does not support FloatType
type yet.")
+getSchema <- function(schema, firstRow = NULL, rdd = NULL) {
+ if (is.null(schema) || (!inherits(schema, "structType") &&
is.null(names(schema)))) {
+ if (is.null(firstRow)) {
+ stopifnot(!is.null(rdd))
+ firstRow <- firstRDD(rdd)
+ }
+ names <- if (is.null(schema)) {
+ names(firstRow)
+ } else {
+ as.list(schema)
+ }
+ if (is.null(names)) {
+ names <- lapply(1:length(firstRow), function(x) {
+ paste("_", as.character(x), sep = "")
+ })
}
+
+ # SPAKR-SQL does not support '.' in column name, so replace it with '_'
+ # TODO(davies): remove this once SPARK-2775 is fixed
+ names <- lapply(names, function(n) {
+ nn <- gsub("[.]", "_", n)
+ if (nn != n) {
+ warning(paste("Use", nn, "instead of", n, " as column name"))
Review comment:
existing: `" as column name"` --> `"as column name"`
don't need the space with paste()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
