wombatu-kun commented on code in PR #20521:
URL: https://github.com/apache/hudi/pull/20521#discussion_r4480319808
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala:
##########
@@ -17,26 +17,58 @@
package org.apache.spark.sql.hudi.dataskipping.rules
-import org.apache.spark.sql.catalyst.expressions.{DateFormatClass, Expression,
FromUnixTime, ParseToDate, ParseToTimestamp, ToUnixTimestamp}
+import org.apache.spark.sql.catalyst.expressions.{DateFormatClass, Expression,
FromUnixTime, Literal, ParseToDate, ParseToTimestamp, ToUnixTimestamp}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step}
+import scala.util.Try
+
/**
* Date/time formatting and parsing: `DateFormatClass`, `FromUnixTime`,
`ToUnixTimestamp`,
* `ParseToDate` and `ParseToTimestamp`. The two parsing expressions have
different constructor
* arities across Spark versions, so they are matched by type and step into
`left` (the parsed
* input, the first constructor field on every supported version) rather than
by extractor.
*
- * Shared guard: none; ANY format is accepted today, though most (for ex,
'dd-MM-yyyy') do not
- * preserve ordering (#19481). Parsing is `partial`: unparseable input yields
NULL.
+ * Guard: `DateFormatClass` and `FromUnixTime` need [[isZoneSafePattern]];
past that, ANY format is
+ * accepted today, though most (for ex, 'dd-MM-yyyy') do not preserve ordering
(#19481). Parsing is
+ * `partial`: unparseable input yields NULL.
*/
object DateTimeFormatRule extends RangeRule {
override def steps(term: Expression): Seq[Step] = term match {
- case DateFormatClass(timestamp, _, _) => Seq(Step(term, timestamp,
Increasing, fileGuard = None, partial = false))
- case FromUnixTime(seconds, _, _) => Seq(Step(term, seconds, Increasing,
fileGuard = None, partial = false))
+ // NOTE: date_format and from_unixtime render the time in the session time
zone, so a pattern field that
+ // depends on the time of day or on the offset (for ex, HH, mm or
XXX) only keeps the ordering when the
+ // zone's offset never changes: local time runs backwards across a
DST fall-back (for ex, 05:30Z is
+ // 01:30 EDT but the later 06:10Z is 01:10 EST). A date-only pattern
is assumed unaffected: the local
+ // date only runs backwards where the clocks fall back across
midnight, which tzdb last records in 2010
+ // (America/St_Johns at 00:01)
Review Comment:
Done 5d2de096d593
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
