[
https://issues.apache.org/jira/browse/SPARK-26248?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16722616#comment-16722616
]
ASF GitHub Bot commented on SPARK-26248:
----------------------------------------
asfgit closed pull request #23202: [SPARK-26248][SQL] Infer date type from CSV
URL: https://github.com/apache/spark/pull/23202
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
index 35ade136cc607..11f3740d99a72 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchema.scala
@@ -22,16 +22,20 @@ import scala.util.control.Exception.allCatch
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.analysis.TypeCoercion
import org.apache.spark.sql.catalyst.expressions.ExprUtils
-import org.apache.spark.sql.catalyst.util.TimestampFormatter
+import org.apache.spark.sql.catalyst.util.{DateFormatter, TimestampFormatter}
import org.apache.spark.sql.types._
class CSVInferSchema(val options: CSVOptions) extends Serializable {
@transient
- private lazy val timestampParser = TimestampFormatter(
+ private lazy val timestampFormatter = TimestampFormatter(
options.timestampFormat,
options.timeZone,
options.locale)
+ @transient
+ private lazy val dateFormatter = DateFormatter(
+ options.dateFormat,
+ options.locale)
private val decimalParser = {
ExprUtils.getDecimalParser(options.locale)
@@ -104,6 +108,7 @@ class CSVInferSchema(val options: CSVOptions) extends
Serializable {
compatibleType(typeSoFar,
tryParseDecimal(field)).getOrElse(StringType)
case DoubleType => tryParseDouble(field)
case TimestampType => tryParseTimestamp(field)
+ case DateType => tryParseDate(field)
case BooleanType => tryParseBoolean(field)
case StringType => StringType
case other: DataType =>
@@ -159,9 +164,16 @@ class CSVInferSchema(val options: CSVOptions) extends
Serializable {
}
private def tryParseTimestamp(field: String): DataType = {
- // This case infers a custom `dataFormat` is set.
- if ((allCatch opt timestampParser.parse(field)).isDefined) {
+ if ((allCatch opt timestampFormatter.parse(field)).isDefined) {
TimestampType
+ } else {
+ tryParseDate(field)
+ }
+ }
+
+ private def tryParseDate(field: String): DataType = {
+ if ((allCatch opt dateFormatter.parse(field)).isDefined) {
+ DateType
} else {
tryParseBoolean(field)
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
index c2b525ad1a9f8..84b2e616a4426 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/csv/CSVInferSchemaSuite.scala
@@ -187,4 +187,22 @@ class CSVInferSchemaSuite extends SparkFunSuite with
SQLHelper {
Seq("en-US", "ko-KR", "ru-RU", "de-DE").foreach(checkDecimalInfer(_,
DecimalType(7, 0)))
}
+
+ test("inferring date type") {
+ var options = new CSVOptions(Map("dateFormat" -> "yyyy/MM/dd"), false,
"GMT")
+ var inferSchema = new CSVInferSchema(options)
+ assert(inferSchema.inferField(NullType, "2018/12/02") == DateType)
+
+ options = new CSVOptions(Map("dateFormat" -> "MMM yyyy"), false, "GMT")
+ inferSchema = new CSVInferSchema(options)
+ assert(inferSchema.inferField(NullType, "Dec 2018") == DateType)
+
+ options = new CSVOptions(
+ Map("dateFormat" -> "yyyy-MM-dd", "timestampFormat" ->
"yyyy-MM-dd'T'HH:mm:ss"),
+ columnPruning = false,
+ defaultTimeZoneId = "GMT")
+ inferSchema = new CSVInferSchema(options)
+ assert(inferSchema.inferField(NullType, "2018-12-03T11:00:00") ==
TimestampType)
+ assert(inferSchema.inferField(NullType, "2018-12-03") == DateType)
+ }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Infer date type from CSV
> ------------------------
>
> Key: SPARK-26248
> URL: https://issues.apache.org/jira/browse/SPARK-26248
> Project: Spark
> Issue Type: Improvement
> Components: SQL
> Affects Versions: 2.4.0
> Reporter: Maxim Gekk
> Assignee: Maxim Gekk
> Priority: Minor
> Fix For: 3.0.0
>
>
> Currently, DateType cannot be inferred from CSV. To parse CSV string, you
> have to specify schema explicitly if CSV input contains dates. This ticket
> aims to extend CSVInferSchema to support such inferring.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
