This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20526-57abf057f968e3d4f9caa9b61d7c6269d5620a63 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 5630de96a10e220bdd8313268ec11cfa8c963ae2 Author: Vova Kolmakov <[email protected]> AuthorDate: Tue Nov 10 09:44:12 2026 +0000 fix(spark): guard data skipping on a timestamp's local date against fall-backs across midnight (#20526) * fix(spark): guard data skipping on a timestamp's local date against fall-backs across midnight * addressed review comments: move the local date reversal guard into fileGuards on the rendering arms * addressed review comments: move localDateReversals to TimeZones, read derived guard bounds through TryEval in Step, lead the reversal guard with a range check, and sharpen the guard tests --- .../spark/sql/hudi/dataskipping/RangeRule.scala | 31 +++++++- .../spark/sql/hudi/dataskipping/TimeZones.scala | 44 ++++++++++- .../sql/hudi/dataskipping/rules/CastRule.scala | 19 ++++- .../dataskipping/rules/DateTimeFormatRule.scala | 19 +++-- .../hudi/dataskipping/rules/DateTimeOrdering.scala | 47 ++++++++++-- .../data-skipping/golden-index-filters.txt | 38 +++++----- .../org/apache/hudi/TestDataSkippingStages.scala | 74 ++++++++++++++++++- .../org/apache/hudi/TestDataSkippingUtils.scala | 86 ++++++++++++++++++++++ .../hudi/TestHoodieCatalystExpressionUtils.scala | 11 ++- 9 files changed, 319 insertions(+), 50 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala index 3d2a71a89a79..57a2798099f1 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/RangeRule.scala @@ -17,7 +17,7 @@ package org.apache.spark.sql.hudi.dataskipping -import org.apache.spark.sql.catalyst.expressions.{Expression, If, Literal, TryEval} +import org.apache.spark.sql.catalyst.expressions.{And, Expression, If, LeafExpression, Literal, TryEval} import org.apache.spark.sql.hudi.dataskipping.rules.{CastRule, ConstantOperandRule, DateTimeFormatRule, MonotonicFunctionRule, PrefixRule, TimeZoneShiftRule} /** @@ -61,7 +61,8 @@ case object Decreasing extends Direction * @param child the operand whose range is pushed through, for ex `x` * @param direction whether `term` grows or shrinks with `child` * @param fileGuard per-file condition under which the mapping holds (for ex, no overflow); outside it - * the bounds become NULL, i.e. unknown, and [[StatsFilterCompiler]] keeps the file + * the bounds become NULL, i.e. unknown, and [[StatsFilterCompiler]] keeps the file. It + * reads a bound of `child` that fails to evaluate as NULL * @param partial whether `term` can be undefined for a non-NULL `child` (for ex, `log` of a negative, * or a parse of an unparseable string). Spark reports that as NULL or, depending on * the session (for ex, ANSI), as an exception; the rebuilt bounds are wrapped in @@ -71,7 +72,7 @@ case object Decreasing extends Direction case class Step(term: Expression, child: Expression, direction: Direction, - fileGuard: Option[TermRange => Expression], + fileGuard: Option[Step.Guard], partial: Boolean) { def apply(r: TermRange): TermRange = { @@ -89,10 +90,20 @@ case class Step(term: Expression, // NOTE: Uses `term.dataType` (same as the rebuilt bound's) since stats-table bounds are unresolved here private def guarded(bound: Expression, r: TermRange): Expression = fileGuard match { - case Some(guard) => If(guard(r), bound, Literal(null, term.dataType)) + case Some(guard) => + val input = r.copy(lo = readInGuard(r.lo), hi = readInGuard(r.hi)) + If(guard(input), bound, Literal(null, term.dataType)) case None => bound } + // NOTE: The guard reads both bounds, while a one-sided verdict (for ex, `<`) reads one, so a derived bound + // that fails to evaluate (for ex, from_utc_timestamp of the largest timestamp) is NULL there, which + // keeps the file. A leaf (a stats column or a literal) cannot fail, so it is read as is + private def readInGuard(bound: Expression): Expression = bound match { + case leaf: LeafExpression => leaf + case _ => TryEval(bound) + } + // NOTE: TryEval is in every supported Spark version, so no version-specific shape (for ex, a // `failOnError = false` copy of each parse expression) is needed private def nullOnFailure(bound: Expression): Expression = if (partial) TryEval(bound) else bound @@ -109,6 +120,18 @@ case class Step(term: Expression, } } +object Step { + + /** A per-file condition over the range of the step's child, see `fileGuard` of [[Step]]. */ + type Guard = TermRange => Expression + + /** Both guards: the mapping holds where each one does. A missing guard leaves the other as is. */ + def andGuards(a: Option[Guard], b: Option[Guard]): Option[Guard] = (a, b) match { + case (Some(x), Some(y)) => Some(r => And(x(r), y(r))) + case _ => a.orElse(b) + } +} + /** * A family of functions sharing one guard. Not sealed: families live one per file under `rules`. * diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala index 7b84d0d70105..d6e771f7558d 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala @@ -19,11 +19,16 @@ package org.apache.spark.sql.hudi.dataskipping import org.apache.spark.sql.catalyst.util.DateTimeUtils +import java.time.Instant +import java.time.zone.{ZoneOffsetTransition, ZoneRules} +import java.util.concurrent.ConcurrentHashMap + +import scala.collection.JavaConverters._ import scala.util.Try /** - * Time zone checks shared by the rules whose ordering depends on a zone's offset: `TimeZoneShiftRule`, and - * `DateTimeFormatRule` through `DateTimeOrdering`. + * Time zone checks shared by the rules whose ordering depends on a zone's offset: `TimeZoneShiftRule`, + * `DateTimeFormatRule` through `DateTimeOrdering`, and `CastRule` for a timestamp's local date. */ object TimeZones { @@ -32,6 +37,37 @@ object TimeZones { * evaluates it, so for ex +8:00 is +08:00 and the short ID PST is America/Los_Angeles; an unknown zone is not * fixed. */ - def isFixedOffsetZone(timeZoneId: String): Boolean = - Try(DateTimeUtils.getZoneId(timeZoneId).getRules.isFixedOffset).getOrElse(false) + def isFixedOffsetZone(timeZoneId: String): Boolean = rulesOf(timeZoneId).exists(_.isFixedOffset) + + private val localDateReversalsByZoneId = new ConcurrentHashMap[String, Option[Seq[Instant]]]() + + /** + * Instants at which the local date runs backwards in the zone, resolved the way Spark evaluates it and + * cached per zone id (see the overload over `ZoneRules`). None, so that the rendering takes no step, if + * there is no zone (not analyzed), the zone is unknown, or its current rules keep doing it. + */ + def localDateReversals(timeZoneId: Option[String]): Option[Seq[Instant]] = + timeZoneId.flatMap { id => + localDateReversalsByZoneId.computeIfAbsent(id, _ => rulesOf(id).flatMap(rules => localDateReversals(rules))) + } + + /** + * Instants at which the local date runs backwards in a zone with the given rules, that is where its clocks fell back + * across midnight, or None if its current rules keep doing it. Falling back at midnight itself (for ex, Asia/Beirut) + * repeats the previous date without running backwards, since the local date before the transition is that of the + * instant just before it. The instants are in chronological order, as `ZoneRules.getTransitions` lists them + */ + def localDateReversals(rules: ZoneRules): Option[Seq[Instant]] = { + def isReversal(transition: ZoneOffsetTransition): Boolean = transition.isOverlap && + transition.getDateTimeAfter.toLocalDate.isBefore(transition.getDateTimeBefore.minusNanos(1).toLocalDate) + // NOTE: A rule falls back at the same time of day every year, so a single year shows whether it reverses the date + if (rules.getTransitionRules.asScala.exists(rule => isReversal(rule.createTransition(2000)))) { + None + } else { + Some(rules.getTransitions.asScala.filter(isReversal).map(_.getInstant).toList) + } + } + + // The rules of the zone, resolved the way Spark evaluates it; None for an unknown zone + private def rulesOf(timeZoneId: String): Option[ZoneRules] = Try(DateTimeUtils.getZoneId(timeZoneId).getRules).toOption } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala index 7f6f86d5411b..eccb4241bc39 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/CastRule.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.hudi.dataskipping.rules import org.apache.spark.sql.catalyst.expressions.{Cast, Expression} -import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step} +import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step, TimeZones} import org.apache.spark.sql.types.{BinaryType, BooleanType, ByteType, DataType, DateType, DecimalType, DoubleType, FloatType, IntegerType, LongType, NumericType, ShortType, StringType, TimestampNTZType, TimestampType} import scala.util.Try @@ -28,6 +28,7 @@ import scala.util.Try * * Shared guard: [[preservesOrder]] of the child type to the target type. The (from, to) allowlist * lives here, at the top of this object, and is deny-by-default: a cast pair not listed has no range. + * A timestamp to date cast is ranged apart from it, with a per-file guard (see [[steps]]). */ object CastRule extends RangeRule { @@ -49,6 +50,9 @@ object CastRule extends RangeRule { * - string to date/timestamp, since unpadded strings sort differently from what they parse to (for * ex, "2022-1-5" > "2022-01-10" as strings, but parses to an earlier date). Expression indexes on * to_date/to_timestamp do not need it: `ExpressionIndexSupport` strips the function first + * - timestamp to date, since the local date runs backwards where the clocks fell back across + * midnight (for ex, America/St_Johns went from 00:01 on 2009-11-01 back to 23:01 on 2009-10-31). + * [[steps]] ranges it with a per-file guard instead */ private[dataskipping] def preservesOrder(from: DataType, to: DataType): Boolean = (from, to) match { @@ -78,9 +82,8 @@ object CastRule extends RangeRule { case (TimestampType, LongType | FloatType | DoubleType) => true // NOTE: A date maps to the start of that day, which increases with the date case (DateType, TimestampType | TimestampNTZType) => true - // NOTE: Truncating to the (local) date is non-decreasing, also across a DST fall-back since - // the date does not repeat - case (TimestampType | TimestampNTZType, DateType) => true + // NOTE: A timestamp_ntz truncates to its wall-clock date, which never runs backwards + case (TimestampNTZType, DateType) => true // Binary and strings // NOTE: The default (UTF8_BINARY) string compares by its bytes, same as binary. Matching the @@ -92,6 +95,14 @@ object CastRule extends RangeRule { } override def steps(term: Expression): Seq[Step] = term match { + // NOTE: The allowlist rejects this cast: a file spanning an instant at which the local date runs + // backwards takes no bounds, and a zone whose current rules keep doing it takes no step + case Cast(child, DateType, timeZoneId, _) if hasDataType(child) && child.dataType == TimestampType => + TimeZones.localDateReversals(timeZoneId).toSeq.map { instants => + Step(term, child, Increasing, + fileGuard = DateTimeOrdering.noLocalDateReversal(instants, DateTimeOrdering.asTimestamp), partial = false) + } + // NOTE: Some casts (for ex, string to date in non-ANSI mode) return NULL on malformed input; // `partial` is left false for now, as it has no behavioral effect yet case cast @ Cast(child, _, _, _) if hasDataType(child) && preservesOrder(child.dataType, cast.dataType) => diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala index 340e19994389..91c0b92096a5 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql.hudi.dataskipping.rules import org.apache.spark.sql.catalyst.expressions.{And, DateFormatClass, Expression, FromUnixTime, GetTimestamp, GreaterThanOrEqual, LessThan, Literal, ToUnixTimestamp, UnixTimestamp} -import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step, TermRange} +import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step, TermRange, TimeZones} import org.apache.spark.sql.types.{DataType, DateType, LongType, StringType, TimestampNTZType, TimestampType} import java.time.{LocalDate, ZoneOffset} @@ -32,11 +32,12 @@ import scala.util.Try * * Shared guard: [[DateTimeOrdering]] (allowlisted literal format, non-LEGACY parser policy, and for a * time-of-day pattern a fixed-offset time zone), plus a per-file guard on the values the format or - * parse is sound for. No parsing step without a time zone: the analyzer sets one on every resolved - * expression. + * parse is sound for. No parsing or formatting step without a time zone: the analyzer sets one on every + * resolved expression. * * - Formatting (`DateFormatClass`, `FromUnixTime`): the instant lies in years 0001 to 9999 (see - * [[formattableInstants]]). + * [[formattableInstants]]), and the file spans no instant at which the zone's local date runs + * backwards (see [[DateTimeOrdering.noLocalDateReversal]]). * - Parsing a string (`ToUnixTimestamp`, `UnixTimestamp`, `GetTimestamp`): every string of the file * starts with a digit (see [[digitFirstStrings]]). Parsing is `partial`: an unparseable string is * NULL, or throws under ANSI or the EXCEPTION parser policy; `Step` turns a throwing bound into NULL, @@ -53,10 +54,16 @@ object DateTimeFormatRule extends RangeRule { override def steps(term: Expression): Seq[Step] = term match { case DateFormatClass(timestamp, format, timeZoneId) if dataTypeOf(timestamp).contains(TimestampType) && DateTimeOrdering.formatPreservesOrder(format, timeZoneId) => - Seq(Step(term, timestamp, Increasing, fileGuard = Some(formattableInstants), partial = false)) + TimeZones.localDateReversals(timeZoneId).toSeq.map { instants => + Step(term, timestamp, Increasing, fileGuard = Step.andGuards(Some(formattableInstants), + DateTimeOrdering.noLocalDateReversal(instants, DateTimeOrdering.asTimestamp)), partial = false) + } case FromUnixTime(seconds, format, timeZoneId) if dataTypeOf(seconds).contains(LongType) && DateTimeOrdering.formatPreservesOrder(format, timeZoneId) => - Seq(Step(term, seconds, Increasing, fileGuard = Some(formattableSeconds), partial = false)) + TimeZones.localDateReversals(timeZoneId).toSeq.map { instants => + Step(term, seconds, Increasing, fileGuard = Step.andGuards(Some(formattableSeconds), + DateTimeOrdering.noLocalDateReversal(instants, DateTimeOrdering.asEpochSeconds)), partial = false) + } // NOTE: `to_unix_timestamp` and `unix_timestamp` evaluate the same way, and take the same guards case ToUnixTimestamp(time, format, timeZoneId, _) => diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala index 66afeeddf661..884c901e149d 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala @@ -17,15 +17,19 @@ package org.apache.spark.sql.hudi.dataskipping.rules -import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} +import org.apache.spark.sql.catalyst.expressions.{And, Expression, GreaterThanOrEqual, LessThan, Literal, Not, Or} +import org.apache.spark.sql.catalyst.util.DateTimeUtils +import org.apache.spark.sql.hudi.dataskipping.Step.Guard import org.apache.spark.sql.hudi.dataskipping.TimeZones import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StringType +import org.apache.spark.sql.types.{LongType, StringType, TimestampType} import org.apache.spark.unsafe.types.UTF8String +import java.time.Instant + /** - * Shared guards of [[DateTimeFormatRule]]: when does turning a timestamp into text (formatting) or - * text into a timestamp (parsing) preserve ordering? + * Shared guards of [[DateTimeFormatRule]], and of [[CastRule]] for a timestamp's local date: when does + * turning a timestamp into text (formatting) or text into a timestamp (parsing) preserve ordering? * * - Pattern: the format is a literal on the allowlist below. Every allowlisted pattern is year-first * and fixed-width, so the lexicographic order of the strings is the chronological order of the @@ -42,10 +46,11 @@ import org.apache.spark.unsafe.types.UTF8String * EST). Across a spring-forward gap, a wall clock inside the gap is shifted forward past later ones * (for ex, in America/Los_Angeles '2022-03-13 02:30:00' parses after '2022-03-13 03:05:00'). A * date-only parse is monotone in every zone (a gap shifts a start of day by less than a day). - * - * NOTE: A date-only format is assumed monotone in every zone: the local date only runs backwards where - * the clocks fall back across midnight, which tzdb last records in 2010 (America/St_Johns at - * 00:01; see #20525) + * - Zone, a timestamp's local date (formatting, and `CAST(ts AS DATE)`): the local date runs + * backwards where the clocks fell back across midnight (for ex, America/St_Johns went from 00:01 + * on 2009-11-01 back to 23:01 on 2009-10-31). A file spanning such an instant takes no bounds (see + * [[noLocalDateReversal]]), and a zone whose current rules keep doing it takes no step (see + * `TimeZones.localDateReversals`). */ object DateTimeOrdering { @@ -125,4 +130,30 @@ object DateTimeOrdering { // the zone never changes its offset (an unknown zone is not fixed) private def isZoneSafe(p: Pattern, timeZoneId: Option[String]): Boolean = !p.hasTimeOfDay || timeZoneId.exists(TimeZones.isFixedOffsetZone) + + /** + * File guard of rendering a timestamp's local date: the file's range spans none of `instants`, since only + * a file spanning one can hold rows that render outside of [f(lo), f(hi)]. None, i.e. unguarded, without + * instants. + * + * @param instants in chronological order, as `TimeZones.localDateReversals` lists them + * @param toLiteral an instant as a value of the rendered input: [[asTimestamp]] or [[asEpochSeconds]] + */ + def noLocalDateReversal(instants: Seq[Instant], toLiteral: Instant => Literal): Option[Guard] = + if (instants.isEmpty) { + None + } else { + val literals = instants.map(toLiteral) + // NOTE: A file wholly before the first instant or from the last one on spans none, which settles most files + // (for ex, any after 2010) in two comparisons + Some(r => Or( + Or(LessThan(r.hi, literals.head), GreaterThanOrEqual(r.lo, literals.last)), + literals.map(t => Not(And(LessThan(r.lo, t), GreaterThanOrEqual(r.hi, t)))).reduce[Expression](And(_, _)))) + } + + /** An instant as a timestamp, the input of `CAST(ts AS DATE)` and `date_format`. */ + val asTimestamp: Instant => Literal = instant => Literal(DateTimeUtils.instantToMicros(instant), TimestampType) + + /** An instant as seconds since the epoch, the input of `from_unixtime`. */ + val asEpochSeconds: Instant => Literal = instant => Literal(instant.getEpochSecond, LongType) } diff --git a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt index 4ece295497a0..dee69d4ad7b7 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt +++ b/hudi-spark-datasource/hudi-spark/src/test/resources/data-skipping/golden-index-filters.txt @@ -989,28 +989,28 @@ cs startswith(cast(to_date(s) as timestamp), 'ab') true:boolean ei startswith(cast(to_date(s) as timestamp), 'ab') true:boolean cs NOT startswith(cast(to_date(s) as timestamp), 'ab') true:boolean ei NOT startswith(cast(to_date(s) as timestamp), 'ab') true:boolean -cs ((cast(a as bigint) + 1:bigint) = 0:bigint) coalesce(((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= 0:bigint) AND (if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint)), true:boolean) -cs (0:bigint < (cast(a as bigint) + 1:bigint)) coalesce((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) > 0:bigint), true:boolean) -cs ((cast(a as bigint) + 1:bigint) >= 0:bigint) coalesce((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint), true:boolean) -cs NOT ((cast(a as bigint) + 1:bigint) = 0:bigint) coalesce(NOT ((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND (if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)), true:boolean) -cs ((cast(a as bigint) + 1:bigint) = cast(0:bigint as bigint)) coalesce(((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= cast(0:bigint as bigint)) AND (if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= cast(0:bigint as bigint))), true:boolean) -cs (cast(a as bigint) + 1:bigint) IN (0:bigint, null:bigint) coalesce((((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= 0:bigint) AND (if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint)) OR ((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= null:bigint) AND (if((cast(a [...] -cs NOT (cast(a as bigint) + 1:bigint) IN (0:bigint) coalesce(NOT ((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND (if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)), true:boolean) +cs ((cast(a as bigint) + 1:bigint) = 0:bigint) coalesce(((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= 0:bigint) AND (if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint)), true:boolean) +cs (0:bigint < (cast(a as bigint) + 1:bigint)) coalesce((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) > 0:bigint), true:boolean) +cs ((cast(a as bigint) + 1:bigint) >= 0:bigint) coalesce((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint), true:boolean) +cs NOT ((cast(a as bigint) + 1:bigint) = 0:bigint) coalesce(NOT ((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND (if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)), true:boolean) +cs ((cast(a as bigint) + 1:bigint) = cast(0:bigint as bigint)) coalesce(((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= cast(0:bigint as bigint)) AND (if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= cast(0:bigint as bigint))), true:boolean) +cs (cast(a as bigint) + 1:bigint) IN (0:bigint, null:bigint) coalesce((((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= 0:bigint) AND (if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 0:bigint)) OR ((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= n [...] +cs NOT (cast(a as bigint) + 1:bigint) IN (0:bigint) coalesce(NOT ((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) = 0:bigint) AND (if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) = 0:bigint)), true:boolean) cs startswith((cast(a as bigint) + 1:bigint), 'ab') true:boolean -ei startswith((cast(a as bigint) + 1:bigint), 'ab') coalesce(((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= 'ab') AND (if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 'ab')), true:boolean) +ei startswith((cast(a as bigint) + 1:bigint), 'ab') coalesce(((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= 'ab') AND (if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint) >= 'ab')), true:boolean) cs NOT startswith((cast(a as bigint) + 1:bigint), 'ab') true:boolean -ei NOT startswith((cast(a as bigint) + 1:bigint), 'ab') coalesce(NOT (startswith(if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint), 'ab') AND startswith(if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint), 'ab')), true:boolean) -cs ((datediff(d, 19723:date) + 1:int) = 0:int) coalesce(((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND (if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int)), true:boolean) -cs (0:int < (datediff(d, 19723:date) + 1:int)) coalesce((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) > 0:int), true:boolean) -cs ((datediff(d, 19723:date) + 1:int) >= 0:int) coalesce((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int), true:boolean) -cs NOT ((datediff(d, 19723:date) + 1:int) = 0:int) coalesce(NOT ((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND (if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) = 0:int)), true:boolean) -cs ((datediff(d, 19723:date) + 1:int) = cast(0:int as int)) coalesce(((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) <= cast(0:int as int)) AND (if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) >= cast(0:int as int))), true:boolean) -cs (datediff(d, 19723:date) + 1:int) IN (0:int, null:int) coalesce((((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND (if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int)) OR ((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) <= null:int) AND (if((datediff(d_maxValue, 19723:date) <= [...] -cs NOT (datediff(d, 19723:date) + 1:int) IN (0:int) coalesce(NOT ((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND (if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) = 0:int)), true:boolean) +ei NOT startswith((cast(a as bigint) + 1:bigint), 'ab') coalesce(NOT (startswith(if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint), 'ab') AND startswith(if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint) + 1:bigint), null:bigint), 'ab')), true:boolean) +cs ((datediff(d, 19723:date) + 1:int) = 0:int) coalesce(((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND (if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int)), true:boolean) +cs (0:int < (datediff(d, 19723:date) + 1:int)) coalesce((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) > 0:int), true:boolean) +cs ((datediff(d, 19723:date) + 1:int) >= 0:int) coalesce((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int), true:boolean) +cs NOT ((datediff(d, 19723:date) + 1:int) = 0:int) coalesce(NOT ((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND (if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) = 0:int)), true:boolean) +cs ((datediff(d, 19723:date) + 1:int) = cast(0:int as int)) coalesce(((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) <= cast(0:int as int)) AND (if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) >= cast(0:int as int))), true:boolean) +cs (datediff(d, 19723:date) + 1:int) IN (0:int, null:int) coalesce((((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) <= 0:int) AND (if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 0:int)) OR ((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) <= null:int) AND (if((tryeval(d [...] +cs NOT (datediff(d, 19723:date) + 1:int) IN (0:int) coalesce(NOT ((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) = 0:int) AND (if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) = 0:int)), true:boolean) cs startswith((datediff(d, 19723:date) + 1:int), 'ab') true:boolean -ei startswith((datediff(d, 19723:date) + 1:int), 'ab') coalesce(((if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) <= 'ab') AND (if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 'ab')), true:boolean) +ei startswith((datediff(d, 19723:date) + 1:int), 'ab') coalesce(((if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int) <= 'ab') AND (if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int) >= 'ab')), true:boolean) cs NOT startswith((datediff(d, 19723:date) + 1:int), 'ab') true:boolean -ei NOT startswith((datediff(d, 19723:date) + 1:int), 'ab') coalesce(NOT (startswith(if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int), 'ab') AND startswith(if((datediff(d_maxValue, 19723:date) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int), 'ab')), true:boolean) +ei NOT startswith((datediff(d, 19723:date) + 1:int), 'ab') coalesce(NOT (startswith(if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_minValue, 19723:date) + 1:int), null:int), 'ab') AND startswith(if((tryeval(datediff(d_maxValue, 19723:date)) <= 2147483646:int), (datediff(d_maxValue, 19723:date) + 1:int), null:int), 'ab')), true:boolean) cs (concat(s, s) = '') true:boolean cs ('' < concat(s, s)) true:boolean cs (concat(s, s) >= '') true:boolean @@ -1102,7 +1102,7 @@ ei NOT (startswith((a + 1:int), 'ab') AND ((0.000:decimal(13,3) < (dec * 1.5:dec cs (NOT (((from_unixtime(l, 'yyyy-MM-dd') = cast('' as string)) OR NOT (x = 0.0:double)) OR NOT (a + a) IN (0:int)) OR NOT (NOT (NOT startswith(to_date(s), 'ab') AND ((a + 'x') = cast(0:int as int))) OR NOT (NOT startswith((a + b), 'ab') AND dec IS NOT NULL))) true:boolean ei (NOT (((from_unixtime(l, 'yyyy-MM-dd') = cast('' as string)) OR NOT (x = 0.0:double)) OR NOT (a + a) IN (0:int)) OR NOT (NOT (NOT startswith(to_date(s), 'ab') AND ((a + 'x') = cast(0:int as int))) OR NOT (NOT startswith((a + b), 'ab') AND dec IS NOT NULL))) true:boolean cs ((((cast(a as bigint) + 1:bigint) = cast(0:bigint as bigint)) AND (NOT startswith(cast(dec as double), 'ab') AND NOT from_unixtime(l, 'yyyy-MM-dd') IN (''))) AND (NOT (date_add(d, 1:int) = 0:date) OR ((0:int * a) = cast(0:int as int)))) true:boolean -ei ((((cast(a as bigint) + 1:bigint) = cast(0:bigint as bigint)) AND (NOT startswith(cast(dec as double), 'ab') AND NOT from_unixtime(l, 'yyyy-MM-dd') IN (''))) AND (NOT (date_add(d, 1:int) = 0:date) OR ((0:int * a) = cast(0:int as int)))) (coalesce(((if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= cast(0:bigint as bigint)) AND (if((cast(a_maxValue as bigint) <= 9223372036854775806:bigint), (cast(a_maxValue as bigint [...] +ei ((((cast(a as bigint) + 1:bigint) = cast(0:bigint as bigint)) AND (NOT startswith(cast(dec as double), 'ab') AND NOT from_unixtime(l, 'yyyy-MM-dd') IN (''))) AND (NOT (date_add(d, 1:int) = 0:date) OR ((0:int * a) = cast(0:int as int)))) (coalesce(((if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_minValue as bigint) + 1:bigint), null:bigint) <= cast(0:bigint as bigint)) AND (if((tryeval(cast(a_maxValue as bigint)) <= 9223372036854775806:bigint), (cast(a_ [...] cs ((lower(s) = cast('' as string)) AND NOT ((x / -2.0:double) = cast(0.0:double as double))) true:boolean cs NOT (to_date(cast(ts as string)) = 0:date) true:boolean cs NOT (NOT (NOT shiftright(8:int, a) IN (0:int) AND ((cast(a as smallint) >= 0:smallint) AND (0:timestamp < from_utc_timestamp(ts, 'UTC')))) AND NOT (a = 0:int)) true:boolean diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala index 5e1a5ae304a5..c089251b11b1 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala @@ -19,17 +19,24 @@ package org.apache.hudi import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.expressions.{Abs, Add, And, AttributeReference, Coalesce, DateFormatClass, EqualNullSafe, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, If, In, InSet, IsNull, LeafExpression, LessThan, LessThanOrEqual, Literal, Log, Not, Or, StartsWith, TryEval, UnaryMinus} +import org.apache.spark.sql.catalyst.expressions.{Abs, Add, And, AttributeReference, Cast, Coalesce, DateFormatClass, EqualNullSafe, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, If, In, InSet, IsNull, LeafExpression, LessThan, LessThanOrEqual, Literal, Log, Not, Or, StartsWith, TryEval, UnaryMinus} import org.apache.spark.sql.catalyst.expressions.Literal.TrueLiteral -import org.apache.spark.sql.hudi.dataskipping.{ColumnStatsSource, Decreasing, Increasing, Leaf, Op, RangeRule, StatsFilterCompiler, Step, TermRange, Verdict} +import org.apache.spark.sql.hudi.dataskipping.{ColumnStatsSource, Decreasing, Increasing, Leaf, Op, RangeRule, StatsFilterCompiler, Step, TermRange, TimeZones, Verdict} import org.apache.spark.sql.hudi.dataskipping.rules.DateTimeOrdering import org.apache.spark.sql.hudi.dataskipping.rules.DateTimeOrdering.Pattern import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{DataType, DoubleType, IntegerType, StringType, StructField, StructType, TimestampType} +import org.apache.spark.sql.types.{DataType, DateType, DoubleType, IntegerType, StringType, StructField, StructType, TimestampType} import org.apache.spark.unsafe.types.UTF8String import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} import org.junit.jupiter.api.Test +import java.time.{DayOfWeek, Duration, Instant, LocalDate, LocalTime, Month, ZoneId, ZoneOffset} +import java.time.zone.{ZoneOffsetTransition, ZoneOffsetTransitionRule, ZoneRules} +import java.time.zone.ZoneOffsetTransitionRule.TimeDefinition +import java.util.Collections + +import scala.collection.JavaConverters._ + /** * Unit tests of the stages on their own: [[Leaf.parse]], [[Verdict]], [[Step.apply]] and * [[TermRange.derive]] over a stub rule table. End-to-end outputs are pinned by @@ -300,6 +307,12 @@ class TestDataSkippingStages { val ts = AttributeReference("ts", TimestampType)() val formatted = EqualTo(DateFormatClass(ts, Literal("yyyy-MM-dd"), Some("UTC")), Literal("2024-01-01")) assertEquals(true, overNullStats(formatted, ColumnStatsSource(Seq("ts")), TimestampType)) + // A cast to the local date: unguarded in UTC, guarded where the local date has run backwards + // (America/St_Johns) + def castToDateIn(zone: String): Expression = + In(Cast(ts, DateType, Some(zone)), Seq(Literal(LocalDate.of(2009, 10, 31)), Literal(LocalDate.of(2009, 11, 1)))) + assertEquals(null, overNullStats(castToDateIn("UTC"), ColumnStatsSource(Seq("ts")), TimestampType)) + assertEquals(true, overNullStats(castToDateIn("America/St_Johns"), ColumnStatsSource(Seq("ts")), TimestampType)) } @Test @@ -366,6 +379,61 @@ class TestDataSkippingStages { assertFalse(DateTimeOrdering.parsePreservesOrder(Literal("yyyy-M-d"), Some("UTC"), zoneDependent = true)) } + @Test + def testLocalDateReversals(): Unit = { + def reversals(zoneId: String): Option[Seq[Instant]] = TimeZones.localDateReversals(ZoneId.of(zoneId).getRules) + + // America/St_Johns fell back across midnight every year from 1987 to 2010 + val stJohns = reversals("America/St_Johns").get + assertTrue(stJohns.contains(Instant.parse("2009-11-01T02:31:00Z"))) + assertEquals(Instant.parse("2010-11-07T02:31:00Z"), stJohns.last) + // Falling back at midnight itself (Asia/Beirut, or Pacific/Apia repeating a whole day in 1892) repeats the previous + // date without running backwards, and a fall-back within the day (America/New_York) does not change the date + Seq("Asia/Beirut", "Pacific/Apia", "America/New_York", "UTC").foreach { zoneId => + assertEquals(Some(Seq.empty), reversals(zoneId), zoneId) + } + + // A current rule falling back from 00:30 to 23:30 runs the date backwards every year, without bound + val (standard, summer) = (ZoneOffset.ofHours(-4), ZoneOffset.ofHours(-3)) + val rules = ZoneRules.of(standard, standard, + Collections.emptyList[ZoneOffsetTransition](), Collections.emptyList[ZoneOffsetTransition](), Seq( + ZoneOffsetTransitionRule.of(Month.MARCH, 8, DayOfWeek.SUNDAY, LocalTime.of(2, 0), false, TimeDefinition.WALL, + standard, standard, summer), + ZoneOffsetTransitionRule.of(Month.NOVEMBER, 1, DayOfWeek.SUNDAY, LocalTime.of(0, 30), false, TimeDefinition.WALL, + standard, summer, standard)).asJava) + assertEquals(None, TimeZones.localDateReversals(rules)) + + // By zone id, resolved the way Spark evaluates it: the short id CNT is America/St_Johns + assertEquals(Some(stJohns), TimeZones.localDateReversals(Some("CNT"))) + assertEquals(Some(Seq.empty), TimeZones.localDateReversals(Some("UTC"))) + // No time zone (not analyzed) or an unknown one: no step + assertEquals(None, TimeZones.localDateReversals(None)) + assertEquals(None, TimeZones.localDateReversals(Some("Not/A_Zone"))) + } + + @Test + def testNoLocalDateReversalGuard(): Unit = { + // America/Phoenix ran its date backwards at 06:01Z on 1944-01-01 and on 1944-10-01 + val (first, last) = (Instant.parse("1944-01-01T06:01:00Z"), Instant.parse("1944-10-01T06:01:00Z")) + val guard = DateTimeOrdering.noLocalDateReversal(Seq(first, last), DateTimeOrdering.asTimestamp).get + // A bound of the file, None standing for one that failed to evaluate + def bound(instant: Option[Instant]): Expression = + instant.map(DateTimeOrdering.asTimestamp).getOrElse(Literal(null, TimestampType)) + def overFile(lo: Option[Instant], hi: Option[Instant]): Any = + guard(TermRange(bound(lo), bound(hi), None, None, nullBoundIsUnknown = false)).eval() + val minute = Duration.ofMinutes(1) + + // A file ending at an instant spans it, and one starting at it does not + assertEquals(false, overFile(Some(first.minus(minute)), Some(first))) + assertEquals(false, overFile(Some(last.minus(minute)), Some(last))) + assertEquals(true, overFile(Some(last), Some(last.plus(minute)))) + assertEquals(true, overFile(Some(first), Some(last.minus(minute)))) + // Without one bound, the file is known to span none only if the other bound alone rules them all out + assertEquals(true, overFile(None, Some(first.minus(minute)))) + assertEquals(true, overFile(Some(last), None)) + assertEquals(null, overFile(Some(first.minus(minute)), None)) + } + @Test def testDateTimeLetterLiteralIsFormatOnly(): Unit = { // The parser matches 'T' in any case: '2022-03-05t09:00:00' parses, and sorts after diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala index c03d099802fe..7d2926afbde7 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala @@ -346,6 +346,92 @@ class TestDataSkippingUtils extends HoodieSparkClientTestBase with SparkAdapterS } } + @Test + def testLocalDateAcrossMidnightFallBack(): Unit = { + withSqlConf(SESSION_LOCAL_TIMEZONE.key -> "America/St_Johns") { + // At 02:31Z on 2009-11-01 the clocks went from 00:01 NDT back to 23:01 NST of the previous day, so 02:30:30Z + // renders as 2009-11-01 but 02:31Z as 2009-10-31. Re-applying a date rendering over min/max stats puts the max + // below the min and would wrongly prune file_1 for either date; must keep the file + val acrossFallBack = IndexRow("file_1", valueCount = 2, + A_minValue = 1257042630L, A_maxValue = 1257042660L, A_nullCount = 0, // epoch seconds of C's bounds + C_minValue = new Timestamp(1257042630000L), // 2009-11-01 02:30:30Z + C_maxValue = new Timestamp(1257042660000L), // 2009-11-01 02:31:00Z, the first instant after the fall-back + C_nullCount = 0) + // Files that do not span the fall-back are still pruned + val afterFallBack = IndexRow("file_2", valueCount = 2, + A_minValue = 1257042660L, A_maxValue = 1257043200L, A_nullCount = 0, + C_minValue = new Timestamp(1257042660000L), // 2009-11-01 02:31:00Z, renders as 2009-10-31 + C_maxValue = new Timestamp(1257043200000L), // 2009-11-01 02:40:00Z, renders as 2009-10-31 + C_nullCount = 0) + val earlierDay = IndexRow("file_3", valueCount = 2, + A_minValue = 1256904000L, A_maxValue = 1256907600L, A_nullCount = 0, + C_minValue = new Timestamp(1256904000000L), // 2009-10-30 12:00Z + C_maxValue = new Timestamp(1256907600000L), // 2009-10-30 13:00Z + C_nullCount = 0) + val files = Seq(acrossFallBack, afterFallBack, earlierDay) + Seq( + "date_format(C, 'yyyy-MM-dd') = '2009-10-31'" -> Seq("file_1", "file_2"), + "date_format(C, 'yyyy-MM-dd') = '2009-11-01'" -> Seq("file_1"), + "from_unixtime(A, 'yyyy-MM-dd') = '2009-10-31'" -> Seq("file_1", "file_2"), + "CAST(C AS DATE) IN (DATE'2009-10-29', DATE'2009-10-31')" -> Seq("file_1", "file_2"), + "date_add(CAST(C AS DATE), 1) = DATE'2009-11-01'" -> Seq("file_1", "file_2") + ).foreach { case (filter, expected) => + assertEquals(expected, applyFilterExpr(resolveExpr(spark, filter, sourceTableSchema), files), filter) + } + // to_date reaches data skipping as the cast it stands for, once the optimizer replaced it. Two dates: with one, + // the optimizer turns IN into "=", which Spark 3.5+ rewrites into a range on C itself + assertEquals(Seq("file_1", "file_2"), + lookupThroughFullOptimizer("to_date(C) IN (DATE'2009-10-29', DATE'2009-10-31')", files)) + + // The same date at both ends does not rule out a row of the previous date in between + val aroundFallBack = IndexRow("file_4", valueCount = 3, + C_minValue = new Timestamp(1257042630000L), // 2009-11-01 02:30:30Z, renders as 2009-11-01 + C_maxValue = new Timestamp(1257046800000L), // 2009-11-01 03:40:00Z, renders as 2009-11-01 + C_nullCount = 0) + assertEquals(Seq("file_4"), applyFilterExpr( + resolveExpr(spark, "NOT (date_format(C, 'yyyy-MM-dd') = '2009-11-01')", sourceTableSchema), Seq(aroundFallBack))) + } + } + + @Test + def testLocalDateGuardToleratesFailingBounds(): Unit = { + // America/Phoenix ran its date backwards twice in 1944, so the cast to the local date is guarded there, and the + // guard reads both bounds of the cast's input, while "<" only needs the min. The max of each file overflows, + // which must not fail the lookup. Without its max, file_1 may span the first instant, so it is kept: its min + // renders as 1944-01-01, but a row 30 seconds later renders as 1943-12-31. The min of file_2 is past both + // instants, which settles the guard without the max, so file_2 is still pruned + withSqlConf(SESSION_LOCAL_TIMEZONE.key -> "America/Phoenix") { + assertEquals(Seq("file_1"), applyFilterExpr( + resolveExpr(spark, "CAST(from_utc_timestamp(C, '+08:00') AS DATE) < DATE'1944-01-01'", sourceTableSchema), + filesWithFailingMax)) + } + } + + @Test + def testFormatGuardToleratesFailingBounds(): Unit = { + // Formatting is guarded in every zone, to years 0001 to 9999, and that guard reads both bounds too. The lookup + // must not fail; as the guard needs the max, both files are kept + withSqlConf(SESSION_LOCAL_TIMEZONE.key -> "UTC") { + assertEquals(Seq("file_1", "file_2"), applyFilterExpr( + resolveExpr(spark, "date_format(from_utc_timestamp(C, '+08:00'), 'yyyy-MM-dd') < '2000-01-01'", sourceTableSchema), + filesWithFailingMax)) + } + } + + // Shifting the max of these files, the largest timestamp, by from_utc_timestamp overflows + private def filesWithFailingMax: Seq[IndexRow] = { + val failingMax = new Timestamp(Long.MaxValue / 1000) // in year 294247 + Seq( + IndexRow("file_1", valueCount = 2, + C_minValue = new Timestamp(-820547970000L), // 1943-12-31 22:00:30Z, shifted to 06:00:30Z on 1944-01-01 + C_maxValue = failingMax, + C_nullCount = 0), + IndexRow("file_2", valueCount = 2, + C_minValue = new Timestamp(978307200000L), // 2001-01-01 00:00Z + C_maxValue = failingMax, + C_nullCount = 0)) + } + @ParameterizedTest @MethodSource(Array("testStringsLookupFilterExpressionsSource")) def testStringsLookupFilterExpressions(sourceExpr: Expression, input: Seq[IndexRow], output: Seq[String]): Unit = { diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala index 68e597b6516c..93569202e0ce 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala @@ -247,6 +247,8 @@ class TestHoodieCatalystExpressionUtils extends SparkAdapterSupport { assertEquals(Some(tsAttr), matched(DateFormatClass(tsAttr, Literal(pattern), newYork)), pattern) assertEquals(Some(longAttr), matched(FromUnixTime(longAttr, Literal(pattern), newYork)), pattern) } + // In America/St_Johns the local date has run backwards, so a date-only pattern is guarded per file there. + assertEquals(Some(tsAttr), matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd"), Some("America/St_Johns")))) // PST resolves to America/Los_Angeles; Kolkata has no DST but historical offset changes. assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd HH:mm"), Some("PST")))) assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd HH:mm"), Some("Asia/Kolkata")))) @@ -255,6 +257,7 @@ class TestHoodieCatalystExpressionUtils extends SparkAdapterSupport { assertEquals(None, matched(DateFormatClass(tsAttr, Literal(null, StringType), newYork))) assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd HH:mm"), Some("Not/AZone")))) assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd HH:mm"), None))) + assertEquals(None, matched(FromUnixTime(longAttr, Literal("yyyy-MM-dd"), None))) } @Test @@ -342,9 +345,13 @@ class TestHoodieCatalystExpressionUtils extends SparkAdapterSupport { assertEquals(None, matched(Cast(dateAttr, StringType))) assertEquals(None, matched(Cast(dateAttr, IntegerType))) assertEquals(None, matched(Cast(boolAttr, DecimalType(1, 1)))) - // A date maps to the start of its day, and truncating a timestamp to its date is non-decreasing. + // A date maps to the start of its day. assertEquals(Some(dateAttr), matched(Cast(dateAttr, TimestampType))) - assertEquals(Some(tsAttr), matched(Cast(tsAttr, DateType))) + // A timestamp's local date runs backwards where the clocks fell back across midnight, so it is + // ranged with a per-file guard there (America/St_Johns), and not at all without a time zone. + assertEquals(Some(false), guarded(Cast(tsAttr, DateType, Some("UTC")), tsAttr)) + assertEquals(Some(true), guarded(Cast(tsAttr, DateType, Some("America/St_Johns")), tsAttr)) + assertEquals(None, matched(Cast(tsAttr, DateType))) // The default string and binary both compare by bytes. assertEquals(Some(binAttr), matched(Cast(binAttr, StringType))) // Boolean to bigint keeps false < true as 0 < 1, and an identity cast is always allowed.
