Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/23165#discussion_r237532982
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningUtils.scala
---
@@ -147,13 +163,13 @@ object PartitioningUtils {
columnNames.zip(literals).map { case (name, Literal(_, dataType))
=>
// We always assume partition columns are nullable since we've
no idea whether null values
// will be appended in the future.
- StructField(name, dataType, nullable = true)
+ StructField(name, userSpecifiedDataTypes.getOrElse(name,
dataType), nullable = true)
}
}
// Finally, we create `Partition`s based on paths and resolved
partition values.
val partitions =
resolvedPartitionValues.zip(pathsWithPartitionValues).map {
- case (PartitionValues(_, literals), (path, _)) =>
+ case (PartitionValues(columnNames, literals), (path, _)) =>
--- End diff --
unnecessary change?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
