Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/23165#discussion_r237508120
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PartitioningAwareFileIndex.scala
---
@@ -126,33 +126,15 @@ abstract class PartitioningAwareFileIndex(
val caseInsensitiveOptions = CaseInsensitiveMap(parameters)
val timeZoneId =
caseInsensitiveOptions.get(DateTimeUtils.TIMEZONE_OPTION)
.getOrElse(sparkSession.sessionState.conf.sessionLocalTimeZone)
- val inferredPartitionSpec = PartitioningUtils.parsePartitions(
+
+ val caseSensitive = sparkSession.sqlContext.conf.caseSensitiveAnalysis
+ PartitioningUtils.parsePartitions(
leafDirs,
typeInference =
sparkSession.sessionState.conf.partitionColumnTypeInferenceEnabled,
basePaths = basePaths,
+ userSpecifiedSchema = userSpecifiedSchema,
+ caseSensitive = caseSensitive,
timeZoneId = timeZoneId)
- userSpecifiedSchema match {
- case Some(userProvidedSchema) if userProvidedSchema.nonEmpty =>
- val userPartitionSchema =
-
combineInferredAndUserSpecifiedPartitionSchema(inferredPartitionSpec)
--- End diff --
we can remove `combineInferredAndUserSpecifiedPartitionSchema` now
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
