This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 9a7728c5862a fix(spark): guard data skipping on a timestamp's local
date against fall-backs across midnight (#20526)
9a7728c5862a is described below
commit 9a7728c5862a8939f58f47c35abb1d838fc2dcb3
Author: Vova Kolmakov <[email protected]>
AuthorDate: Wed Nov 11 03:21:20 2026 +0000
fix(spark): guard data skipping on a timestamp's local date against
fall-backs across midnight (#20526)
* fix(spark): guard data skipping on a timestamp's local date against
fall-backs across midnight
* addressed review comments: move the local date reversal guard into
fileGuards on the rendering arms
* addressed review comments: move localDateReversals to TimeZones, read
derived guard bounds through TryEval in Step, lead the reversal guard with a
range check, and sharpen the guard tests
---
.../spark/sql/hudi/dataskipping/RangeRule.scala | 31 +++++++-
.../spark/sql/hudi/dataskipping/TimeZones.scala | 44 ++++++++++-
.../sql/hudi/dataskipping/rules/CastRule.scala | 19 ++++-
.../dataskipping/rules/DateTimeFormatRule.scala | 19 +++--
.../hudi/dataskipping/rules/DateTimeOrdering.scala | 47 ++++++++++--
.../data-skipping/golden-index-filters.txt | 28 +++----
.../org/apache/hudi/TestDataSkippingStages.scala | 70 +++++++++++++++++-
.../org/apache/hudi/TestDataSkippingUtils.scala | 86 ++++++++++++++++++++++
.../hudi/TestHoodieCatalystExpressionUtils.scala | 11 ++-
9 files changed, 312 insertions(+), 43 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala
index 3d2a71a89a79..57a2798099f1 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.hudi.dataskipping
-import org.apache.spark.sql.catalyst.expressions.{Expression, If, Literal,
TryEval}
+import org.apache.spark.sql.catalyst.expressions.{And, Expression, If,
LeafExpression, Literal, TryEval}
import org.apache.spark.sql.hudi.dataskipping.rules.{CastRule,
ConstantOperandRule, DateTimeFormatRule, MonotonicFunctionRule, PrefixRule,
TimeZoneShiftRule}
/**
@@ -61,7 +61,8 @@ case object Decreasing extends Direction
* @param child the operand whose range is pushed through, for ex `x`
* @param direction whether `term` grows or shrinks with `child`
* @param fileGuard per-file condition under which the mapping holds (for ex,
no overflow); outside it
- * the bounds become NULL, i.e. unknown, and
[[StatsFilterCompiler]] keeps the file
+ * the bounds become NULL, i.e. unknown, and
[[StatsFilterCompiler]] keeps the file. It
+ * reads a bound of `child` that fails to evaluate as NULL
* @param partial whether `term` can be undefined for a non-NULL `child`
(for ex, `log` of a negative,
* or a parse of an unparseable string). Spark reports that
as NULL or, depending on
* the session (for ex, ANSI), as an exception; the rebuilt
bounds are wrapped in
@@ -71,7 +72,7 @@ case object Decreasing extends Direction
case class Step(term: Expression,
child: Expression,
direction: Direction,
- fileGuard: Option[TermRange => Expression],
+ fileGuard: Option[Step.Guard],
partial: Boolean) {
def apply(r: TermRange): TermRange = {
@@ -89,10 +90,20 @@ case class Step(term: Expression,
// NOTE: Uses `term.dataType` (same as the rebuilt bound's) since
stats-table bounds are unresolved here
private def guarded(bound: Expression, r: TermRange): Expression = fileGuard
match {
- case Some(guard) => If(guard(r), bound, Literal(null, term.dataType))
+ case Some(guard) =>
+ val input = r.copy(lo = readInGuard(r.lo), hi = readInGuard(r.hi))
+ If(guard(input), bound, Literal(null, term.dataType))
case None => bound
}
+ // NOTE: The guard reads both bounds, while a one-sided verdict (for ex,
`<`) reads one, so a derived bound
+ // that fails to evaluate (for ex, from_utc_timestamp of the largest
timestamp) is NULL there, which
+ // keeps the file. A leaf (a stats column or a literal) cannot fail,
so it is read as is
+ private def readInGuard(bound: Expression): Expression = bound match {
+ case leaf: LeafExpression => leaf
+ case _ => TryEval(bound)
+ }
+
// NOTE: TryEval is in every supported Spark version, so no version-specific
shape (for ex, a
// `failOnError = false` copy of each parse expression) is needed
private def nullOnFailure(bound: Expression): Expression = if (partial)
TryEval(bound) else bound
@@ -109,6 +120,18 @@ case class Step(term: Expression,
}
}
+object Step {
+
+ /** A per-file condition over the range of the step's child, see `fileGuard`
of [[Step]]. */
+ type Guard = TermRange => Expression
+
+ /** Both guards: the mapping holds where each one does. A missing guard
leaves the other as is. */
+ def andGuards(a: Option[Guard], b: Option[Guard]): Option[Guard] = (a, b)
match {
+ case (Some(x), Some(y)) => Some(r => And(x(r), y(r)))
+ case _ => a.orElse(b)
+ }
+}
+
/**
* A family of functions sharing one guard. Not sealed: families live one per
file under `rules`.
*
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
index 7b84d0d70105..d6e771f7558d 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
@@ -19,11 +19,16 @@ package org.apache.spark.sql.hudi.dataskipping
import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import java.time.Instant
+import java.time.zone.{ZoneOffsetTransition, ZoneRules}
+import java.util.concurrent.ConcurrentHashMap
+
+import scala.collection.JavaConverters._
import scala.util.Try
/**
- * Time zone checks shared by the rules whose ordering depends on a zone's
offset: `TimeZoneShiftRule`, and
- * `DateTimeFormatRule` through `DateTimeOrdering`.
+ * Time zone checks shared by the rules whose ordering depends on a zone's
offset: `TimeZoneShiftRule`,
+ * `DateTimeFormatRule` through `DateTimeOrdering`, and `CastRule` for a
timestamp's local date.
*/
object TimeZones {
@@ -32,6 +37,37 @@ object TimeZones {
* evaluates it, so for ex +8:00 is +08:00 and the short ID PST is
America/Los_Angeles; an unknown zone is not
* fixed.
*/
- def isFixedOffsetZone(timeZoneId: String): Boolean =
-
Try(DateTimeUtils.getZoneId(timeZoneId).getRules.isFixedOffset).getOrElse(false)
+ def isFixedOffsetZone(timeZoneId: String): Boolean =
rulesOf(timeZoneId).exists(_.isFixedOffset)
+
+ private val localDateReversalsByZoneId = new ConcurrentHashMap[String,
Option[Seq[Instant]]]()
+
+ /**
+ * Instants at which the local date runs backwards in the zone, resolved the
way Spark evaluates it and
+ * cached per zone id (see the overload over `ZoneRules`). None, so that the
rendering takes no step, if
+ * there is no zone (not analyzed), the zone is unknown, or its current
rules keep doing it.
+ */
+ def localDateReversals(timeZoneId: Option[String]): Option[Seq[Instant]] =
+ timeZoneId.flatMap { id =>
+ localDateReversalsByZoneId.computeIfAbsent(id, _ =>
rulesOf(id).flatMap(rules => localDateReversals(rules)))
+ }
+
+ /**
+ * Instants at which the local date runs backwards in a zone with the given
rules, that is where its clocks fell back
+ * across midnight, or None if its current rules keep doing it. Falling back
at midnight itself (for ex, Asia/Beirut)
+ * repeats the previous date without running backwards, since the local date
before the transition is that of the
+ * instant just before it. The instants are in chronological order, as
`ZoneRules.getTransitions` lists them
+ */
+ def localDateReversals(rules: ZoneRules): Option[Seq[Instant]] = {
+ def isReversal(transition: ZoneOffsetTransition): Boolean =
transition.isOverlap &&
+
transition.getDateTimeAfter.toLocalDate.isBefore(transition.getDateTimeBefore.minusNanos(1).toLocalDate)
+ // NOTE: A rule falls back at the same time of day every year, so a single
year shows whether it reverses the date
+ if (rules.getTransitionRules.asScala.exists(rule =>
isReversal(rule.createTransition(2000)))) {
+ None
+ } else {
+
Some(rules.getTransitions.asScala.filter(isReversal).map(_.getInstant).toList)
+ }
+ }
+
+ // The rules of the zone, resolved the way Spark evaluates it; None for an
unknown zone
+ private def rulesOf(timeZoneId: String): Option[ZoneRules] =
Try(DateTimeUtils.getZoneId(timeZoneId).getRules).toOption
}
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
index 144737c5ac9c..ae388e7894ca 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.hudi.dataskipping.rules
import org.apache.spark.sql.catalyst.expressions.{Cast, Expression}
-import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step}
+import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step,
TimeZones}
import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType,
DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType,
NumericType, ShortType, StringType, TimestampNTZType, TimestampType}
import scala.util.Try
@@ -28,6 +28,7 @@ import scala.util.Try
*
* Shared guard: [[preservesOrder]] of the child type to the target type. The
(from, to) allowlist
* lives here, at the top of this object, and is deny-by-default: a cast pair
not listed has no range.
+ * A timestamp to date cast is ranged apart from it, with a per-file guard
(see [[steps]]).
*/
object CastRule extends RangeRule {
@@ -49,6 +50,9 @@ object CastRule extends RangeRule {
* - string to date/timestamp, since unpadded strings sort differently from
what they parse to (for
* ex, "2022-1-5" > "2022-01-10" as strings, but parses to an earlier
date). Expression indexes on
* to_date/to_timestamp do not need it: [[TermRange]] asks the source for
the cast itself first
+ * - timestamp to date, since the local date runs backwards where the
clocks fell back across
+ * midnight (for ex, America/St_Johns went from 00:01 on 2009-11-01 back
to 23:01 on 2009-10-31).
+ * [[steps]] ranges it with a per-file guard instead
*/
private[dataskipping] def preservesOrder(from: DataType, to: DataType):
Boolean =
(from, to) match {
@@ -78,9 +82,8 @@ object CastRule extends RangeRule {
case (TimestampType, LongType | FloatType | DoubleType) => true
// NOTE: A date maps to the start of that day, which increases with the
date
case (DateType, TimestampType | TimestampNTZType) => true
- // NOTE: Truncating to the (local) date is non-decreasing, also across a
DST fall-back since
- // the date does not repeat
- case (TimestampType | TimestampNTZType, DateType) => true
+ // NOTE: A timestamp_ntz truncates to its wall-clock date, which never
runs backwards
+ case (TimestampNTZType, DateType) => true
// Binary and strings
// NOTE: The default (UTF8_BINARY) string compares by its bytes, same as
binary. Matching the
@@ -92,6 +95,14 @@ object CastRule extends RangeRule {
}
override def steps(term: Expression): Seq[Step] = term match {
+ // NOTE: The allowlist rejects this cast: a file spanning an instant at
which the local date runs
+ // backwards takes no bounds, and a zone whose current rules keep
doing it takes no step
+ case Cast(child, DateType, timeZoneId, _) if hasDataType(child) &&
child.dataType == TimestampType =>
+ TimeZones.localDateReversals(timeZoneId).toSeq.map { instants =>
+ Step(term, child, Increasing,
+ fileGuard = DateTimeOrdering.noLocalDateReversal(instants,
DateTimeOrdering.asTimestamp), partial = false)
+ }
+
// NOTE: Some casts (for ex, string to date in non-ANSI mode) return NULL
on malformed input;
// `partial` is left false for now, as it has no behavioral effect
yet
case cast @ Cast(child, _, _, _) if hasDataType(child) &&
preservesOrder(child.dataType, cast.dataType) =>
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
index 340e19994389..91c0b92096a5 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.hudi.dataskipping.rules
import org.apache.spark.sql.catalyst.expressions.{And, DateFormatClass,
Expression, FromUnixTime, GetTimestamp, GreaterThanOrEqual, LessThan, Literal,
ToUnixTimestamp, UnixTimestamp}
-import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step,
TermRange}
+import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step,
TermRange, TimeZones}
import org.apache.spark.sql.types.{DataType, DateType, LongType, StringType,
TimestampNTZType, TimestampType}
import java.time.{LocalDate, ZoneOffset}
@@ -32,11 +32,12 @@ import scala.util.Try
*
* Shared guard: [[DateTimeOrdering]] (allowlisted literal format, non-LEGACY
parser policy, and for a
* time-of-day pattern a fixed-offset time zone), plus a per-file guard on the
values the format or
- * parse is sound for. No parsing step without a time zone: the analyzer sets
one on every resolved
- * expression.
+ * parse is sound for. No parsing or formatting step without a time zone: the
analyzer sets one on every
+ * resolved expression.
*
* - Formatting (`DateFormatClass`, `FromUnixTime`): the instant lies in
years 0001 to 9999 (see
- * [[formattableInstants]]).
+ * [[formattableInstants]]), and the file spans no instant at which the
zone's local date runs
+ * backwards (see [[DateTimeOrdering.noLocalDateReversal]]).
* - Parsing a string (`ToUnixTimestamp`, `UnixTimestamp`, `GetTimestamp`):
every string of the file
* starts with a digit (see [[digitFirstStrings]]). Parsing is `partial`:
an unparseable string is
* NULL, or throws under ANSI or the EXCEPTION parser policy; `Step` turns
a throwing bound into NULL,
@@ -53,10 +54,16 @@ object DateTimeFormatRule extends RangeRule {
override def steps(term: Expression): Seq[Step] = term match {
case DateFormatClass(timestamp, format, timeZoneId)
if dataTypeOf(timestamp).contains(TimestampType) &&
DateTimeOrdering.formatPreservesOrder(format, timeZoneId) =>
- Seq(Step(term, timestamp, Increasing, fileGuard =
Some(formattableInstants), partial = false))
+ TimeZones.localDateReversals(timeZoneId).toSeq.map { instants =>
+ Step(term, timestamp, Increasing, fileGuard =
Step.andGuards(Some(formattableInstants),
+ DateTimeOrdering.noLocalDateReversal(instants,
DateTimeOrdering.asTimestamp)), partial = false)
+ }
case FromUnixTime(seconds, format, timeZoneId)
if dataTypeOf(seconds).contains(LongType) &&
DateTimeOrdering.formatPreservesOrder(format, timeZoneId) =>
- Seq(Step(term, seconds, Increasing, fileGuard =
Some(formattableSeconds), partial = false))
+ TimeZones.localDateReversals(timeZoneId).toSeq.map { instants =>
+ Step(term, seconds, Increasing, fileGuard =
Step.andGuards(Some(formattableSeconds),
+ DateTimeOrdering.noLocalDateReversal(instants,
DateTimeOrdering.asEpochSeconds)), partial = false)
+ }
// NOTE: `to_unix_timestamp` and `unix_timestamp` evaluate the same way,
and take the same guards
case ToUnixTimestamp(time, format, timeZoneId, _) =>
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
index 66afeeddf661..884c901e149d 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
@@ -17,15 +17,19 @@
package org.apache.spark.sql.hudi.dataskipping.rules
-import org.apache.spark.sql.catalyst.expressions.{Expression, Literal}
+import org.apache.spark.sql.catalyst.expressions.{And, Expression,
GreaterThanOrEqual, LessThan, Literal, Not, Or}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.hudi.dataskipping.Step.Guard
import org.apache.spark.sql.hudi.dataskipping.TimeZones
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.StringType
+import org.apache.spark.sql.types.{LongType, StringType, TimestampType}
import org.apache.spark.unsafe.types.UTF8String
+import java.time.Instant
+
/**
- * Shared guards of [[DateTimeFormatRule]]: when does turning a timestamp into
text (formatting) or
- * text into a timestamp (parsing) preserve ordering?
+ * Shared guards of [[DateTimeFormatRule]], and of [[CastRule]] for a
timestamp's local date: when does
+ * turning a timestamp into text (formatting) or text into a timestamp
(parsing) preserve ordering?
*
* - Pattern: the format is a literal on the allowlist below. Every
allowlisted pattern is year-first
* and fixed-width, so the lexicographic order of the strings is the
chronological order of the
@@ -42,10 +46,11 @@ import org.apache.spark.unsafe.types.UTF8String
* EST). Across a spring-forward gap, a wall clock inside the gap is
shifted forward past later ones
* (for ex, in America/Los_Angeles '2022-03-13 02:30:00' parses after
'2022-03-13 03:05:00'). A
* date-only parse is monotone in every zone (a gap shifts a start of day
by less than a day).
- *
- * NOTE: A date-only format is assumed monotone in every zone: the local date
only runs backwards where
- * the clocks fall back across midnight, which tzdb last records in 2010
(America/St_Johns at
- * 00:01; see #20525)
+ * - Zone, a timestamp's local date (formatting, and `CAST(ts AS DATE)`): the
local date runs
+ * backwards where the clocks fell back across midnight (for ex,
America/St_Johns went from 00:01
+ * on 2009-11-01 back to 23:01 on 2009-10-31). A file spanning such an
instant takes no bounds (see
+ * [[noLocalDateReversal]]), and a zone whose current rules keep doing it
takes no step (see
+ * `TimeZones.localDateReversals`).
*/
object DateTimeOrdering {
@@ -125,4 +130,30 @@ object DateTimeOrdering {
// the zone never changes its offset (an unknown zone is not fixed)
private def isZoneSafe(p: Pattern, timeZoneId: Option[String]): Boolean =
!p.hasTimeOfDay || timeZoneId.exists(TimeZones.isFixedOffsetZone)
+
+ /**
+ * File guard of rendering a timestamp's local date: the file's range spans
none of `instants`, since only
+ * a file spanning one can hold rows that render outside of [f(lo), f(hi)].
None, i.e. unguarded, without
+ * instants.
+ *
+ * @param instants in chronological order, as
`TimeZones.localDateReversals` lists them
+ * @param toLiteral an instant as a value of the rendered input:
[[asTimestamp]] or [[asEpochSeconds]]
+ */
+ def noLocalDateReversal(instants: Seq[Instant], toLiteral: Instant =>
Literal): Option[Guard] =
+ if (instants.isEmpty) {
+ None
+ } else {
+ val literals = instants.map(toLiteral)
+ // NOTE: A file wholly before the first instant or from the last one on
spans none, which settles most files
+ // (for ex, any after 2010) in two comparisons
+ Some(r => Or(
+ Or(LessThan(r.hi, literals.head), GreaterThanOrEqual(r.lo,
literals.last)),
+ literals.map(t => Not(And(LessThan(r.lo, t), GreaterThanOrEqual(r.hi,
t)))).reduce[Expression](And(_, _))))
+ }
+
+ /** An instant as a timestamp, the input of `CAST(ts AS DATE)` and
`date_format`. */
+ val asTimestamp: Instant => Literal = instant =>
Literal(DateTimeUtils.instantToMicros(instant), TimestampType)
+
+ /** An instant as seconds since the epoch, the input of `from_unixtime`. */
+ val asEpochSeconds: Instant => Literal = instant =>
Literal(instant.getEpochSecond, LongType)
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
index ac02db7cb444..3f3ff8950463 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
+++
b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt
@@ -812,22 +812,22 @@ cs cast(to_date(s) as timestamp) IN (0:timestamp,
null:timestamp) true:boolean
cs NOT cast(to_date(s) as timestamp) IN (0:timestamp) true:boolean
cs startswith(cast(to_date(s) as timestamp), 'ab') true:boolean
cs NOT startswith(cast(to_date(s) as timestamp), 'ab') true:boolean
-cs ((cast(a as bigint) + 1:bigint) = 0:bigint)
coalesce(((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_minValue as bigint) + 1:bigint), null:bigint) <= 0:bigint) AND
(if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint)),
true:boolean)
-cs (0:bigint < (cast(a as bigint) + 1:bigint))
coalesce((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) > 0:bigint), true:boolean)
-cs ((cast(a as bigint) + 1:bigint) >= 0:bigint)
coalesce((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint),
true:boolean)
-cs NOT ((cast(a as bigint) + 1:bigint) = 0:bigint) coalesce(NOT
((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND
(if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)),
true:boolean)
-cs ((cast(a as bigint) + 1:bigint) = cast(0:bigint as bigint))
coalesce(((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_minValue as bigint) + 1:bigint), null:bigint) <= cast(0:bigint as
bigint)) AND (if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= cast(0:bigint as
bigint))), true:boolean)
-cs (cast(a as bigint) + 1:bigint) IN (0:bigint, null:bigint)
coalesce((((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_minValue as bigint) + 1:bigint), null:bigint) <= 0:bigint) AND
(if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint)) OR
((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_minValue as bigint) + 1:bigint), null:bigint) <= null:bigint) AND
(if((cast(a [...]
-cs NOT (cast(a as bigint) + 1:bigint) IN (0:bigint) coalesce(NOT
((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND
(if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint),
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)),
true:boolean)
+cs ((cast(a as bigint) + 1:bigint) = 0:bigint)
coalesce(((if((tryeval(cast(a_maxValue as bigint)) <=
9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint),
null:bigint) <= 0:bigint) AND (if((tryeval(cast(a_maxValue as bigint)) <=
9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint),
null:bigint) >= 0:bigint)), true:boolean)
+cs (0:bigint < (cast(a as bigint) + 1:bigint))
coalesce((if((tryeval(cast(a_maxValue as bigint)) <=
9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint),
null:bigint) > 0:bigint), true:boolean)
+cs ((cast(a as bigint) + 1:bigint) >= 0:bigint)
coalesce((if((tryeval(cast(a_maxValue as bigint)) <=
9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint),
null:bigint) >= 0:bigint), true:boolean)
+cs NOT ((cast(a as bigint) + 1:bigint) = 0:bigint) coalesce(NOT
((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint),
(cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND
(if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint),
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)),
true:boolean)
+cs ((cast(a as bigint) + 1:bigint) = cast(0:bigint as bigint))
coalesce(((if((tryeval(cast(a_maxValue as bigint)) <=
9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint),
null:bigint) <= cast(0:bigint as bigint)) AND (if((tryeval(cast(a_maxValue as
bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) +
1:bigint), null:bigint) >= cast(0:bigint as bigint))), true:boolean)
+cs (cast(a as bigint) + 1:bigint) IN (0:bigint, null:bigint)
coalesce((((if((tryeval(cast(a_maxValue as bigint)) <=
9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint),
null:bigint) <= 0:bigint) AND (if((tryeval(cast(a_maxValue as bigint)) <=
9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint),
null:bigint) >= 0:bigint)) OR ((if((tryeval(cast(a_maxValue as bigint)) <=
9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint),
null:bigint) <= n [...]
+cs NOT (cast(a as bigint) + 1:bigint) IN (0:bigint) coalesce(NOT
((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint),
(cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND
(if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint),
(cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)),
true:boolean)
cs startswith((cast(a as bigint) + 1:bigint), 'ab') true:boolean
cs NOT startswith((cast(a as bigint) + 1:bigint), 'ab') true:boolean
-cs ((datediff(d, 19723:date) + 1:int) = 0:int)
coalesce(((if((datediff(d_maxValue, 19723:date) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND
(if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue,
19723:date) + 1:int), null:int) >= 0:int)), true:boolean)
-cs (0:int < (datediff(d, 19723:date) + 1:int))
coalesce((if((datediff(d_maxValue, 19723:date) <= 2147483646:int),
(datediff(d_maxValue, 19723:date) + 1:int), null:int) > 0:int), true:boolean)
-cs ((datediff(d, 19723:date) + 1:int) >= 0:int)
coalesce((if((datediff(d_maxValue, 19723:date) <= 2147483646:int),
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int), true:boolean)
-cs NOT ((datediff(d, 19723:date) + 1:int) = 0:int) coalesce(NOT
((if((datediff(d_maxValue, 19723:date) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND
(if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue,
19723:date) + 1:int), null:int) = 0:int)), true:boolean)
-cs ((datediff(d, 19723:date) + 1:int) = cast(0:int as int))
coalesce(((if((datediff(d_maxValue, 19723:date) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= cast(0:int as int))
AND (if((datediff(d_maxValue, 19723:date) <= 2147483646:int),
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= cast(0:int as int))),
true:boolean)
-cs (datediff(d, 19723:date) + 1:int) IN (0:int, null:int)
coalesce((((if((datediff(d_maxValue, 19723:date) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND
(if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue,
19723:date) + 1:int), null:int) >= 0:int)) OR ((if((datediff(d_maxValue,
19723:date) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int),
null:int) <= null:int) AND (if((datediff(d_maxValue, 19723:date) <= [...]
-cs NOT (datediff(d, 19723:date) + 1:int) IN (0:int) coalesce(NOT
((if((datediff(d_maxValue, 19723:date) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND
(if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue,
19723:date) + 1:int), null:int) = 0:int)), true:boolean)
+cs ((datediff(d, 19723:date) + 1:int) = 0:int)
coalesce(((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND
(if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int)), true:boolean)
+cs (0:int < (datediff(d, 19723:date) + 1:int))
coalesce((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_maxValue, 19723:date) + 1:int), null:int) > 0:int), true:boolean)
+cs ((datediff(d, 19723:date) + 1:int) >= 0:int)
coalesce((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int), true:boolean)
+cs NOT ((datediff(d, 19723:date) + 1:int) = 0:int) coalesce(NOT
((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND
(if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_maxValue, 19723:date) + 1:int), null:int) = 0:int)), true:boolean)
+cs ((datediff(d, 19723:date) + 1:int) = cast(0:int as int))
coalesce(((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= cast(0:int as int))
AND (if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= cast(0:int as int))),
true:boolean)
+cs (datediff(d, 19723:date) + 1:int) IN (0:int, null:int)
coalesce((((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND
(if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int)) OR
((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) <= null:int) AND
(if((tryeval(d [...]
+cs NOT (datediff(d, 19723:date) + 1:int) IN (0:int) coalesce(NOT
((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND
(if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int),
(datediff(d_maxValue, 19723:date) + 1:int), null:int) = 0:int)), true:boolean)
cs startswith((datediff(d, 19723:date) + 1:int), 'ab') true:boolean
cs NOT startswith((datediff(d, 19723:date) + 1:int), 'ab') true:boolean
cs (concat(s, s) = '') true:boolean
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
index 7fde323f01e5..cd5d9b1c4935 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
@@ -21,7 +21,7 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.expressions.{Abs, Add, And,
AttributeReference, Cast, Coalesce, DateFormatClass, EqualNullSafe, EqualTo,
Expression, FromUnixTime, GreaterThan, GreaterThanOrEqual, Hour, If, In, InSet,
IsNull, LeafExpression, Length, LessThan, LessThanOrEqual, Literal, Log,
Minute, Not, Or, StartsWith, TryEval, UnaryMinus, UnixTimestamp, Upper}
import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral
-import org.apache.spark.sql.hudi.dataskipping.{ColumnStatsSource, Decreasing,
ExpressionIndexSource, Increasing, IndexedExpression, Leaf, Op, RangeRule,
StatsFilterCompiler, Step, TermRange, Verdict}
+import org.apache.spark.sql.hudi.dataskipping.{ColumnStatsSource, Decreasing,
ExpressionIndexSource, Increasing, IndexedExpression, Leaf, Op, RangeRule,
StatsFilterCompiler, Step, TermRange, TimeZones, Verdict}
import org.apache.spark.sql.hudi.dataskipping.rules.DateTimeOrdering
import org.apache.spark.sql.hudi.dataskipping.rules.DateTimeOrdering.Pattern
import org.apache.spark.sql.internal.SQLConf
@@ -30,6 +30,13 @@ import org.apache.spark.unsafe.types.UTF8String
import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
import org.junit.jupiter.api.Test
+import java.time.{DayOfWeek, Duration, Instant, LocalDate, LocalTime, Month,
ZoneId, ZoneOffset}
+import java.time.zone.{ZoneOffsetTransition, ZoneOffsetTransitionRule,
ZoneRules}
+import java.time.zone.ZoneOffsetTransitionRule.TimeDefinition
+import java.util.Collections
+
+import scala.collection.JavaConverters._
+
/**
* Unit tests of the stages on their own: [[Leaf.parse]], [[Verdict]],
[[Step.apply]] and
* [[TermRange.derive]] over a stub rule table. End-to-end outputs are pinned
by
@@ -394,6 +401,12 @@ class TestDataSkippingStages {
val ts = AttributeReference("ts", TimestampType)()
val formatted = EqualTo(DateFormatClass(ts, Literal("yyyy-MM-dd"),
Some("UTC")), Literal("2024-01-01"))
assertEquals(true, overNullStats(formatted, ColumnStatsSource(Seq("ts")),
TimestampType))
+ // A cast to the local date: unguarded in UTC, guarded where the local
date has run backwards
+ // (America/St_Johns)
+ def castToDateIn(zone: String): Expression =
+ In(Cast(ts, DateType, Some(zone)), Seq(Literal(LocalDate.of(2009, 10,
31)), Literal(LocalDate.of(2009, 11, 1))))
+ assertEquals(null, overNullStats(castToDateIn("UTC"),
ColumnStatsSource(Seq("ts")), TimestampType))
+ assertEquals(true, overNullStats(castToDateIn("America/St_Johns"),
ColumnStatsSource(Seq("ts")), TimestampType))
}
@Test
@@ -460,6 +473,61 @@ class TestDataSkippingStages {
assertFalse(DateTimeOrdering.parsePreservesOrder(Literal("yyyy-M-d"),
Some("UTC"), zoneDependent = true))
}
+ @Test
+ def testLocalDateReversals(): Unit = {
+ def reversals(zoneId: String): Option[Seq[Instant]] =
TimeZones.localDateReversals(ZoneId.of(zoneId).getRules)
+
+ // America/St_Johns fell back across midnight every year from 1987 to 2010
+ val stJohns = reversals("America/St_Johns").get
+ assertTrue(stJohns.contains(Instant.parse("2009-11-01T02:31:00Z")))
+ assertEquals(Instant.parse("2010-11-07T02:31:00Z"), stJohns.last)
+ // Falling back at midnight itself (Asia/Beirut, or Pacific/Apia repeating
a whole day in 1892) repeats the previous
+ // date without running backwards, and a fall-back within the day
(America/New_York) does not change the date
+ Seq("Asia/Beirut", "Pacific/Apia", "America/New_York", "UTC").foreach {
zoneId =>
+ assertEquals(Some(Seq.empty), reversals(zoneId), zoneId)
+ }
+
+ // A current rule falling back from 00:30 to 23:30 runs the date backwards
every year, without bound
+ val (standard, summer) = (ZoneOffset.ofHours(-4), ZoneOffset.ofHours(-3))
+ val rules = ZoneRules.of(standard, standard,
+ Collections.emptyList[ZoneOffsetTransition](),
Collections.emptyList[ZoneOffsetTransition](), Seq(
+ ZoneOffsetTransitionRule.of(Month.MARCH, 8, DayOfWeek.SUNDAY,
LocalTime.of(2, 0), false, TimeDefinition.WALL,
+ standard, standard, summer),
+ ZoneOffsetTransitionRule.of(Month.NOVEMBER, 1, DayOfWeek.SUNDAY,
LocalTime.of(0, 30), false, TimeDefinition.WALL,
+ standard, summer, standard)).asJava)
+ assertEquals(None, TimeZones.localDateReversals(rules))
+
+ // By zone id, resolved the way Spark evaluates it: the short id CNT is
America/St_Johns
+ assertEquals(Some(stJohns), TimeZones.localDateReversals(Some("CNT")))
+ assertEquals(Some(Seq.empty), TimeZones.localDateReversals(Some("UTC")))
+ // No time zone (not analyzed) or an unknown one: no step
+ assertEquals(None, TimeZones.localDateReversals(None))
+ assertEquals(None, TimeZones.localDateReversals(Some("Not/A_Zone")))
+ }
+
+ @Test
+ def testNoLocalDateReversalGuard(): Unit = {
+ // America/Phoenix ran its date backwards at 06:01Z on 1944-01-01 and on
1944-10-01
+ val (first, last) = (Instant.parse("1944-01-01T06:01:00Z"),
Instant.parse("1944-10-01T06:01:00Z"))
+ val guard = DateTimeOrdering.noLocalDateReversal(Seq(first, last),
DateTimeOrdering.asTimestamp).get
+ // A bound of the file, None standing for one that failed to evaluate
+ def bound(instant: Option[Instant]): Expression =
+ instant.map(DateTimeOrdering.asTimestamp).getOrElse(Literal(null,
TimestampType))
+ def overFile(lo: Option[Instant], hi: Option[Instant]): Any =
+ guard(TermRange(bound(lo), bound(hi), None, None, nullBoundIsUnknown =
false)).eval()
+ val minute = Duration.ofMinutes(1)
+
+ // A file ending at an instant spans it, and one starting at it does not
+ assertEquals(false, overFile(Some(first.minus(minute)), Some(first)))
+ assertEquals(false, overFile(Some(last.minus(minute)), Some(last)))
+ assertEquals(true, overFile(Some(last), Some(last.plus(minute))))
+ assertEquals(true, overFile(Some(first), Some(last.minus(minute))))
+ // Without one bound, the file is known to span none only if the other
bound alone rules them all out
+ assertEquals(true, overFile(None, Some(first.minus(minute))))
+ assertEquals(true, overFile(Some(last), None))
+ assertEquals(null, overFile(Some(first.minus(minute)), None))
+ }
+
@Test
def testDateTimeLetterLiteralIsFormatOnly(): Unit = {
// The parser matches 'T' in any case: '2022-03-05t09:00:00' parses, and
sorts after
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
index c03d099802fe..7d2926afbde7 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
@@ -346,6 +346,92 @@ class TestDataSkippingUtils extends
HoodieSparkClientTestBase with SparkAdapterS
}
}
+ @Test
+ def testLocalDateAcrossMidnightFallBack(): Unit = {
+ withSqlConf(SESSION_LOCAL_TIMEZONE.key -> "America/St_Johns") {
+ // At 02:31Z on 2009-11-01 the clocks went from 00:01 NDT back to 23:01
NST of the previous day, so 02:30:30Z
+ // renders as 2009-11-01 but 02:31Z as 2009-10-31. Re-applying a date
rendering over min/max stats puts the max
+ // below the min and would wrongly prune file_1 for either date; must
keep the file
+ val acrossFallBack = IndexRow("file_1", valueCount = 2,
+ A_minValue = 1257042630L, A_maxValue = 1257042660L, A_nullCount = 0,
// epoch seconds of C's bounds
+ C_minValue = new Timestamp(1257042630000L), // 2009-11-01 02:30:30Z
+ C_maxValue = new Timestamp(1257042660000L), // 2009-11-01 02:31:00Z,
the first instant after the fall-back
+ C_nullCount = 0)
+ // Files that do not span the fall-back are still pruned
+ val afterFallBack = IndexRow("file_2", valueCount = 2,
+ A_minValue = 1257042660L, A_maxValue = 1257043200L, A_nullCount = 0,
+ C_minValue = new Timestamp(1257042660000L), // 2009-11-01 02:31:00Z,
renders as 2009-10-31
+ C_maxValue = new Timestamp(1257043200000L), // 2009-11-01 02:40:00Z,
renders as 2009-10-31
+ C_nullCount = 0)
+ val earlierDay = IndexRow("file_3", valueCount = 2,
+ A_minValue = 1256904000L, A_maxValue = 1256907600L, A_nullCount = 0,
+ C_minValue = new Timestamp(1256904000000L), // 2009-10-30 12:00Z
+ C_maxValue = new Timestamp(1256907600000L), // 2009-10-30 13:00Z
+ C_nullCount = 0)
+ val files = Seq(acrossFallBack, afterFallBack, earlierDay)
+ Seq(
+ "date_format(C, 'yyyy-MM-dd') = '2009-10-31'" -> Seq("file_1",
"file_2"),
+ "date_format(C, 'yyyy-MM-dd') = '2009-11-01'" -> Seq("file_1"),
+ "from_unixtime(A, 'yyyy-MM-dd') = '2009-10-31'" -> Seq("file_1",
"file_2"),
+ "CAST(C AS DATE) IN (DATE'2009-10-29', DATE'2009-10-31')" ->
Seq("file_1", "file_2"),
+ "date_add(CAST(C AS DATE), 1) = DATE'2009-11-01'" -> Seq("file_1",
"file_2")
+ ).foreach { case (filter, expected) =>
+ assertEquals(expected, applyFilterExpr(resolveExpr(spark, filter,
sourceTableSchema), files), filter)
+ }
+ // to_date reaches data skipping as the cast it stands for, once the
optimizer replaced it. Two dates: with one,
+ // the optimizer turns IN into "=", which Spark 3.5+ rewrites into a
range on C itself
+ assertEquals(Seq("file_1", "file_2"),
+ lookupThroughFullOptimizer("to_date(C) IN (DATE'2009-10-29',
DATE'2009-10-31')", files))
+
+ // The same date at both ends does not rule out a row of the previous
date in between
+ val aroundFallBack = IndexRow("file_4", valueCount = 3,
+ C_minValue = new Timestamp(1257042630000L), // 2009-11-01 02:30:30Z,
renders as 2009-11-01
+ C_maxValue = new Timestamp(1257046800000L), // 2009-11-01 03:40:00Z,
renders as 2009-11-01
+ C_nullCount = 0)
+ assertEquals(Seq("file_4"), applyFilterExpr(
+ resolveExpr(spark, "NOT (date_format(C, 'yyyy-MM-dd') =
'2009-11-01')", sourceTableSchema), Seq(aroundFallBack)))
+ }
+ }
+
+ @Test
+ def testLocalDateGuardToleratesFailingBounds(): Unit = {
+ // America/Phoenix ran its date backwards twice in 1944, so the cast to
the local date is guarded there, and the
+ // guard reads both bounds of the cast's input, while "<" only needs the
min. The max of each file overflows,
+ // which must not fail the lookup. Without its max, file_1 may span the
first instant, so it is kept: its min
+ // renders as 1944-01-01, but a row 30 seconds later renders as
1943-12-31. The min of file_2 is past both
+ // instants, which settles the guard without the max, so file_2 is still
pruned
+ withSqlConf(SESSION_LOCAL_TIMEZONE.key -> "America/Phoenix") {
+ assertEquals(Seq("file_1"), applyFilterExpr(
+ resolveExpr(spark, "CAST(from_utc_timestamp(C, '+08:00') AS DATE) <
DATE'1944-01-01'", sourceTableSchema),
+ filesWithFailingMax))
+ }
+ }
+
+ @Test
+ def testFormatGuardToleratesFailingBounds(): Unit = {
+ // Formatting is guarded in every zone, to years 0001 to 9999, and that
guard reads both bounds too. The lookup
+ // must not fail; as the guard needs the max, both files are kept
+ withSqlConf(SESSION_LOCAL_TIMEZONE.key -> "UTC") {
+ assertEquals(Seq("file_1", "file_2"), applyFilterExpr(
+ resolveExpr(spark, "date_format(from_utc_timestamp(C, '+08:00'),
'yyyy-MM-dd') < '2000-01-01'", sourceTableSchema),
+ filesWithFailingMax))
+ }
+ }
+
+ // Shifting the max of these files, the largest timestamp, by
from_utc_timestamp overflows
+ private def filesWithFailingMax: Seq[IndexRow] = {
+ val failingMax = new Timestamp(Long.MaxValue / 1000) // in year 294247
+ Seq(
+ IndexRow("file_1", valueCount = 2,
+ C_minValue = new Timestamp(-820547970000L), // 1943-12-31 22:00:30Z,
shifted to 06:00:30Z on 1944-01-01
+ C_maxValue = failingMax,
+ C_nullCount = 0),
+ IndexRow("file_2", valueCount = 2,
+ C_minValue = new Timestamp(978307200000L), // 2001-01-01 00:00Z
+ C_maxValue = failingMax,
+ C_nullCount = 0))
+ }
+
@ParameterizedTest
@MethodSource(Array("testStringsLookupFilterExpressionsSource"))
def testStringsLookupFilterExpressions(sourceExpr: Expression, input:
Seq[IndexRow], output: Seq[String]): Unit = {
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
index 68e597b6516c..93569202e0ce 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -247,6 +247,8 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
assertEquals(Some(tsAttr), matched(DateFormatClass(tsAttr,
Literal(pattern), newYork)), pattern)
assertEquals(Some(longAttr), matched(FromUnixTime(longAttr,
Literal(pattern), newYork)), pattern)
}
+ // In America/St_Johns the local date has run backwards, so a date-only
pattern is guarded per file there.
+ assertEquals(Some(tsAttr), matched(DateFormatClass(tsAttr,
Literal("yyyy-MM-dd"), Some("America/St_Johns"))))
// PST resolves to America/Los_Angeles; Kolkata has no DST but historical
offset changes.
assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd
HH:mm"), Some("PST"))))
assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd
HH:mm"), Some("Asia/Kolkata"))))
@@ -255,6 +257,7 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
assertEquals(None, matched(DateFormatClass(tsAttr, Literal(null,
StringType), newYork)))
assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd
HH:mm"), Some("Not/AZone"))))
assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd
HH:mm"), None)))
+ assertEquals(None, matched(FromUnixTime(longAttr, Literal("yyyy-MM-dd"),
None)))
}
@Test
@@ -342,9 +345,13 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
assertEquals(None, matched(Cast(dateAttr, StringType)))
assertEquals(None, matched(Cast(dateAttr, IntegerType)))
assertEquals(None, matched(Cast(boolAttr, DecimalType(1, 1))))
- // A date maps to the start of its day, and truncating a timestamp to its
date is non-decreasing.
+ // A date maps to the start of its day.
assertEquals(Some(dateAttr), matched(Cast(dateAttr, TimestampType)))
- assertEquals(Some(tsAttr), matched(Cast(tsAttr, DateType)))
+ // A timestamp's local date runs backwards where the clocks fell back
across midnight, so it is
+ // ranged with a per-file guard there (America/St_Johns), and not at all
without a time zone.
+ assertEquals(Some(false), guarded(Cast(tsAttr, DateType, Some("UTC")),
tsAttr))
+ assertEquals(Some(true), guarded(Cast(tsAttr, DateType,
Some("America/St_Johns")), tsAttr))
+ assertEquals(None, matched(Cast(tsAttr, DateType)))
// The default string and binary both compare by bytes.
assertEquals(Some(binAttr), matched(Cast(binAttr, StringType)))
// Boolean to bigint keeps false < true as 0 < 1, and an identity cast is
always allowed.