yaooqinn commented on a change in pull request #28674:
URL: https://github.com/apache/spark/pull/28674#discussion_r433100740
##########
File path:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeFormatterHelper.scala
##########
@@ -31,26 +33,92 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.LegacyBehaviorPolicy._
trait DateTimeFormatterHelper {
- private def getOrDefault(accessor: TemporalAccessor, field: ChronoField,
default: Int): Int = {
+ private def getFieldValue(
+ accessor: TemporalAccessor,
+ field: TemporalField): Option[Int] = {
if (accessor.isSupported(field)) {
- accessor.get(field)
+ try {
+ Option(accessor.get(field))
+ } catch {
+ case NonFatal(_) => None
+ }
} else {
- default
+ None
}
}
- protected def toLocalDate(accessor: TemporalAccessor): LocalDate = {
+ @throws[DateTimeException]
+ protected def toLocalDate(accessor: TemporalAccessor, locale: Locale):
LocalDate = {
val localDate = accessor.query(TemporalQueries.localDate())
- // If all the date fields are specified, return the local date directly.
+ // If all the date fields are resolved(yMd or Ywu), return the local date
directly.
if (localDate != null) return localDate
- // Users may want to parse only a few datetime fields from a string and
extract these fields
- // later, and we should provide default values for missing fields.
- // To be compatible with Spark 2.4, we pick 1970 as the default value of
year.
- val year = getOrDefault(accessor, ChronoField.YEAR, 1970)
- val month = getOrDefault(accessor, ChronoField.MONTH_OF_YEAR, 1)
- val day = getOrDefault(accessor, ChronoField.DAY_OF_MONTH, 1)
- LocalDate.of(year, month, day)
+ var res = LocalDate.of(1970, 1, 1)
+
+ val weekFields = WeekFields.of(locale)
+ val weekBasedYearField = weekFields.weekBasedYear
+ var weekBasedYearEnabled = false
+
+ val year = getFieldValue(accessor, weekBasedYearField).map { y =>
Review comment:
```
spark.sql.legacy.timeParserPolicy legacy
spark-sql> select to_timestamp('2020 2020', 'YYYY yyyy');
2020-01-01 00:00:00
spark-sql> select to_timestamp('2020 2020', 'yyyy YYYY');
2019-12-29 00:00:00
spark-sql> select to_timestamp('2020 2020 1', 'YYYY yyyy w');
NULL
spark-sql> select to_timestamp('2020 2020 1', 'yyyy YYYY w');
2019-12-29 00:00:00
```
For corner cases like these, I do not know how to follow
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
