Jonathancui123 commented on code in PR #36871:
URL: https://github.com/apache/spark/pull/36871#discussion_r904402104
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala:
##########
@@ -110,15 +116,43 @@ class CSVInferSchema(val options: CSVOptions) extends
Serializable {
def inferField(typeSoFar: DataType, field: String): DataType = {
if (field == null || field.isEmpty || field == options.nullValue) {
typeSoFar
+ } else
+ if (options.inferDate) {
+ val typeElemInfer = typeSoFar match {
+ case NullType => tryParseInteger(field)
+ case IntegerType => tryParseInteger(field)
+ case LongType => tryParseLong(field)
+ case _: DecimalType => tryParseDecimal(field)
+ case DoubleType => tryParseDouble(field)
+ case DateType => tryParseDateTime(field)
+ case TimestampNTZType => tryParseDateTime(field)
+ case TimestampType => tryParseDateTime(field)
+ case BooleanType => tryParseBoolean(field)
+ case StringType => StringType
+ case other: DataType =>
+ throw QueryExecutionErrors.dataTypeUnexpectedError(other)
+ }
+ compatibleType(typeSoFar, typeElemInfer).getOrElse(StringType)
} else {
val typeElemInfer = typeSoFar match {
case NullType => tryParseInteger(field)
case IntegerType => tryParseInteger(field)
case LongType => tryParseLong(field)
case _: DecimalType => tryParseDecimal(field)
case DoubleType => tryParseDouble(field)
- case TimestampNTZType => tryParseTimestampNTZ(field)
- case TimestampType => tryParseTimestamp(field)
+ case DateType => tryParseDateTime(field)
+ case TimestampNTZType =>
+ if (options.inferDate) {
Review Comment:
My bad, I meant to remove the first match expression but missed it during
code cleanup
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
