This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 4cad2902ff15 fix(spark): guard date_format/from_unixtime data skipping
against DST in the session zone (#20521)
4cad2902ff15 is described below
commit 4cad2902ff15c2ee24f5a8963dfc07a3637ef863
Author: Vova Kolmakov <[email protected]>
AuthorDate: Fri Nov 6 10:36:00 2026 +0700
fix(spark): guard date_format/from_unixtime data skipping against DST in
the session zone (#20521)
* fix(spark): guard date_format/from_unixtime data skipping against DST in
the session zone
* addressed review comments: share the fixed-offset zone check, pin every
non-date pattern letter, merge the DST fall-back tests and link #20525
* addressed review comments: reduce the formatting zone check to the
allowlist's hasTimeOfDay over the shared TimeZones helper
---
.../spark/sql/hudi/dataskipping/TimeZones.scala | 37 ++++++++++++++++++++++
.../dataskipping/rules/DateTimeFormatRule.scala | 10 +++---
.../hudi/dataskipping/rules/DateTimeOrdering.scala | 31 +++++++++---------
.../dataskipping/rules/TimeZoneShiftRule.scala | 11 ++-----
.../org/apache/hudi/TestDataSkippingStages.scala | 19 +++++++----
.../org/apache/hudi/TestDataSkippingUtils.scala | 22 ++++++++++++-
.../hudi/TestHoodieCatalystExpressionUtils.scala | 27 ++++++++++++++++
7 files changed, 121 insertions(+), 36 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
new file mode 100644
index 000000000000..7b84d0d70105
--- /dev/null
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/TimeZones.scala
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hudi.dataskipping
+
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+
+import scala.util.Try
+
+/**
+ * Time zone checks shared by the rules whose ordering depends on a zone's
offset: `TimeZoneShiftRule`, and
+ * `DateTimeFormatRule` through `DateTimeOrdering`.
+ */
+object TimeZones {
+
+ /**
+ * Whether the zone never changes its offset, neither for DST nor
historically. It is resolved the way Spark
+ * evaluates it, so for ex +8:00 is +08:00 and the short ID PST is
America/Los_Angeles; an unknown zone is not
+ * fixed.
+ */
+ def isFixedOffsetZone(timeZoneId: String): Boolean =
+
Try(DateTimeUtils.getZoneId(timeZoneId).getRules.isFixedOffset).getOrElse(false)
+}
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
index 44fc6eba7e49..340e19994389 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeFormatRule.scala
@@ -31,7 +31,7 @@ import scala.util.Try
* `UnixTimestamp` and `GetTimestamp`.
*
* Shared guard: [[DateTimeOrdering]] (allowlisted literal format, non-LEGACY
parser policy, and for a
- * time-of-day parse a fixed-offset time zone), plus a per-file guard on the
values the format or
+ * time-of-day pattern a fixed-offset time zone), plus a per-file guard on the
values the format or
* parse is sound for. No parsing step without a time zone: the analyzer sets
one on every resolved
* expression.
*
@@ -51,11 +51,11 @@ import scala.util.Try
object DateTimeFormatRule extends RangeRule {
override def steps(term: Expression): Seq[Step] = term match {
- case DateFormatClass(timestamp, format, _)
- if dataTypeOf(timestamp).contains(TimestampType) &&
DateTimeOrdering.formatPreservesOrder(format) =>
+ case DateFormatClass(timestamp, format, timeZoneId)
+ if dataTypeOf(timestamp).contains(TimestampType) &&
DateTimeOrdering.formatPreservesOrder(format, timeZoneId) =>
Seq(Step(term, timestamp, Increasing, fileGuard =
Some(formattableInstants), partial = false))
- case FromUnixTime(seconds, format, _)
- if dataTypeOf(seconds).contains(LongType) &&
DateTimeOrdering.formatPreservesOrder(format) =>
+ case FromUnixTime(seconds, format, timeZoneId)
+ if dataTypeOf(seconds).contains(LongType) &&
DateTimeOrdering.formatPreservesOrder(format, timeZoneId) =>
Seq(Step(term, seconds, Increasing, fileGuard =
Some(formattableSeconds), partial = false))
// NOTE: `to_unix_timestamp` and `unix_timestamp` evaluate the same way,
and take the same guards
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
index b5008b52284c..66afeeddf661 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/DateTimeOrdering.scala
@@ -18,13 +18,11 @@
package org.apache.spark.sql.hudi.dataskipping.rules
import org.apache.spark.sql.catalyst.expressions.{Expression, Literal}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.hudi.dataskipping.TimeZones
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.StringType
import org.apache.spark.unsafe.types.UTF8String
-import scala.util.Try
-
/**
* Shared guards of [[DateTimeFormatRule]]: when does turning a timestamp into
text (formatting) or
* text into a timestamp (parsing) preserve ordering?
@@ -38,13 +36,16 @@ import scala.util.Try
* - Policy: `spark.sql.legacy.timeParserPolicy` is not LEGACY. LEGACY parses
with SimpleDateFormat,
* which is lenient (for ex, '2022-3-6', '2022-03-06x' and ' 2022-03-06'
all parse with
* 'yyyy-MM-dd'), so parseable strings are no longer fixed-width.
- * - Zone, parsing only: a time-of-day pattern parsed in the session time
zone needs a fixed-offset
- * zone. Across a spring-forward gap, a wall clock inside the gap is
shifted forward past later ones
+ * - Zone: a time-of-day pattern needs a fixed-offset session time zone, both
to format and to parse.
+ * Across a fall-back, local time runs backwards, so a later instant can
format before an earlier
+ * one (for ex, in America/New_York 05:30Z formats as '01:30' EDT, but the
later 06:10Z as '01:10'
+ * EST). Across a spring-forward gap, a wall clock inside the gap is
shifted forward past later ones
* (for ex, in America/Los_Angeles '2022-03-13 02:30:00' parses after
'2022-03-13 03:05:00'). A
* date-only parse is monotone in every zone (a gap shifts a start of day
by less than a day).
*
- * NOTE: The zone side of formatting is guarded elsewhere: a time-of-day
pattern in a DST session
- * zone by #20521, a local date that runs backwards at a fall-back
across midnight by #20526
+ * NOTE: A date-only format is assumed monotone in every zone: the local date
only runs backwards where
+ * the clocks fall back across midnight, which tzdb last records in 2010
(America/St_Johns at
+ * 00:01; see #20525)
*/
object DateTimeOrdering {
@@ -105,9 +106,9 @@ object DateTimeOrdering {
// NOTE: An enum conf from Spark 4.1 on, a string conf before, so it is
compared as a string
"LEGACY".equalsIgnoreCase(SQLConf.get.getConf(SQLConf.LEGACY_TIME_PARSER_POLICY).toString)
- /** Whether formatting with `format` preserves ordering (see the object
scaladoc). */
- def formatPreservesOrder(format: Expression): Boolean =
- !isLegacyTimeParserPolicy && pattern(format).isDefined
+ /** Whether formatting with `format` in `timeZoneId` preserves ordering (see
the object scaladoc). */
+ def formatPreservesOrder(format: Expression, timeZoneId: Option[String]):
Boolean =
+ !isLegacyTimeParserPolicy && pattern(format).exists(isZoneSafe(_,
timeZoneId))
/**
* Whether parsing with `format` in `timeZoneId` preserves ordering (see the
object scaladoc).
@@ -117,11 +118,11 @@ object DateTimeOrdering {
*/
def parsePreservesOrder(format: Expression, timeZoneId: Option[String],
zoneDependent: Boolean): Boolean =
!isLegacyTimeParserPolicy && timeZoneId.isDefined &&
pattern(format).exists { p =>
- !p.hasLetterLiteral && (!zoneDependent || !p.hasTimeOfDay ||
timeZoneId.exists(isFixedOffsetZone))
+ !p.hasLetterLiteral && (!zoneDependent || isZoneSafe(p, timeZoneId))
}
- // Whether the zone, resolved the way Spark evaluates it, never changes its
offset. An unknown zone is
- // not fixed
- private def isFixedOffsetZone(timeZoneId: String): Boolean =
-
Try(DateTimeUtils.getZoneId(timeZoneId).getRules.isFixedOffset).getOrElse(false)
+ // Whether the session time zone cannot reorder what `p` formats or parses:
`p` has no time of day, or
+ // the zone never changes its offset (an unknown zone is not fixed)
+ private def isZoneSafe(p: Pattern, timeZoneId: Option[String]): Boolean =
+ !p.hasTimeOfDay || timeZoneId.exists(TimeZones.isFixedOffsetZone)
}
diff --git
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
index fc2a4fce0fee..2296bb125110 100644
---
a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
+++
b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/dataskipping/rules/TimeZoneShiftRule.scala
@@ -18,10 +18,7 @@
package org.apache.spark.sql.hudi.dataskipping.rules
import org.apache.spark.sql.catalyst.expressions.{Expression,
FromUTCTimestamp, Literal, ToUTCTimestamp}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step}
-
-import scala.util.Try
+import org.apache.spark.sql.hudi.dataskipping.{Increasing, RangeRule, Step,
TimeZones}
/**
* Time zone shifts: `FromUTCTimestamp`, `ToUTCTimestamp`.
@@ -45,11 +42,9 @@ object TimeZoneShiftRule extends RangeRule {
}
// Only a literal zone is matched, since the optimizer folds a constant zone
to a literal before data
- // skipping runs. It is resolved the way Spark evaluates it, so for ex +8:00
is +08:00 and the short
- // ID PST is America/Los_Angeles; an unknown zone is not matched
+ // skipping runs; `TimeZones.isFixedOffsetZone` resolves it the way Spark
evaluates it
private def isFixedOffsetTimeZone(expr: Expression): Boolean = expr match {
- case Literal(timeZone, _) if timeZone != null =>
-
Try(DateTimeUtils.getZoneId(timeZone.toString).getRules.isFixedOffset).getOrElse(false)
+ case Literal(timeZone, _) if timeZone != null =>
TimeZones.isFixedOffsetZone(timeZone.toString)
case _ => false
}
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
index bd84d4afddd8..5e1a5ae304a5 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingStages.scala
@@ -335,9 +335,10 @@ class TestDataSkippingStages {
@Test
def testDateTimeZoneGuards(): Unit = {
- // UTC and fixed offsets never shift, so a time-of-day parse is accepted
+ // UTC and fixed offsets never shift, so a time-of-day parse or format is
accepted
Seq("UTC", "+08:00", "-05:30", "GMT+8").foreach { zone =>
assertTrue(DateTimeOrdering.parsePreservesOrder(timeOfDay, Some(zone),
zoneDependent = true), zone)
+ assertTrue(DateTimeOrdering.formatPreservesOrder(timeOfDay, Some(zone)),
zone)
}
// America/Los_Angeles skips 02:00 to 03:00 when springing forward, so a
time-of-day parse is
@@ -348,17 +349,21 @@ class TestDataSkippingStages {
assertTrue(DateTimeOrdering.parsePreservesOrder(timeOfDay, losAngeles,
zoneDependent = false))
// ... while a date-only parse is monotone in every zone
assertTrue(DateTimeOrdering.parsePreservesOrder(dateOnly, losAngeles,
zoneDependent = true))
+ // It falls back from 02:00 to 01:00, so a time-of-day format is refused
too, and a date-only one is not
+ assertFalse(DateTimeOrdering.formatPreservesOrder(timeOfDay, losAngeles))
+ assertTrue(DateTimeOrdering.formatPreservesOrder(dateOnly, losAngeles))
// Asia/Kolkata has had no DST since 1945, but its offset changed, so it
is not fixed
assertFalse(DateTimeOrdering.parsePreservesOrder(timeOfDay,
Some("Asia/Kolkata"), zoneDependent = true))
+ assertFalse(DateTimeOrdering.formatPreservesOrder(timeOfDay,
Some("Asia/Kolkata")))
// No time zone (not analyzed) or an unknown one
assertFalse(DateTimeOrdering.parsePreservesOrder(dateOnly, None,
zoneDependent = false))
assertFalse(DateTimeOrdering.parsePreservesOrder(timeOfDay,
Some("Not/A_Zone"), zoneDependent = true))
+ assertFalse(DateTimeOrdering.formatPreservesOrder(timeOfDay, None))
+ assertFalse(DateTimeOrdering.formatPreservesOrder(timeOfDay,
Some("Not/A_Zone")))
// A pattern off the allowlist, in any zone
- assertFalse(DateTimeOrdering.formatPreservesOrder(Literal("MM/dd/yyyy")))
+ assertFalse(DateTimeOrdering.formatPreservesOrder(Literal("MM/dd/yyyy"),
Some("UTC")))
assertFalse(DateTimeOrdering.parsePreservesOrder(Literal("yyyy-M-d"),
Some("UTC"), zoneDependent = true))
- // Formatting takes no zone guard here (#20521, #20526)
- assertTrue(DateTimeOrdering.formatPreservesOrder(timeOfDay))
}
@Test
@@ -367,7 +372,7 @@ class TestDataSkippingStages {
// '2022-03-05T23:00:00'. Formatting always writes 'T'
assertFalse(DateTimeOrdering.parsePreservesOrder(isoTimeOfDay,
Some("UTC"), zoneDependent = true))
assertFalse(DateTimeOrdering.parsePreservesOrder(Literal("yyyy-MM-dd'T'HH:mm:ss.SSS"),
Some("UTC"), zoneDependent = false))
- assertTrue(DateTimeOrdering.formatPreservesOrder(isoTimeOfDay))
+ assertTrue(DateTimeOrdering.formatPreservesOrder(isoTimeOfDay,
Some("UTC")))
}
@Test
@@ -379,12 +384,12 @@ class TestDataSkippingStages {
}
// SimpleDateFormat parses leniently (for ex, '2022-3-6' with 'yyyy-MM-dd')
withPolicy("LEGACY") {
- assertFalse(DateTimeOrdering.formatPreservesOrder(dateOnly))
+ assertFalse(DateTimeOrdering.formatPreservesOrder(dateOnly, Some("UTC")))
assertFalse(DateTimeOrdering.parsePreservesOrder(dateOnly, Some("UTC"),
zoneDependent = true))
}
Seq("CORRECTED", "EXCEPTION").foreach { policy =>
withPolicy(policy) {
- assertTrue(DateTimeOrdering.formatPreservesOrder(dateOnly), policy)
+ assertTrue(DateTimeOrdering.formatPreservesOrder(dateOnly,
Some("UTC")), policy)
assertTrue(DateTimeOrdering.parsePreservesOrder(dateOnly, Some("UTC"),
zoneDependent = true), policy)
}
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
index 9cee99e41114..1d804facbca8 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestDataSkippingUtils.scala
@@ -138,7 +138,7 @@ class TestDataSkippingUtils extends
HoodieSparkClientTestBase with SparkAdapterS
}
@Test
- def testTimestampCastsAcrossDstFallBack(): Unit = {
+ def testSessionZoneConversionsAcrossDstFallBack(): Unit = {
spark.sqlContext.setConf(SESSION_LOCAL_TIMEZONE.key, "America/New_York")
try {
// 05:30Z formats as 01:30 EDT but the later 06:10Z formats as 01:10
EST, so the string max
@@ -146,6 +146,10 @@ class TestDataSkippingUtils extends
HoodieSparkClientTestBase with SparkAdapterS
// file_1 even though its 01:30 row matches; must fall back to
always-true, keeping the file
val input = Seq(
IndexRow("file_1", valueCount = 2,
+ // epoch seconds of C's bounds
+ A_minValue = TestDataSkippingUtils.NyFallBackEdt0130.getTime / 1000,
+ A_maxValue = TestDataSkippingUtils.NyFallBackEst0110.getTime / 1000,
+ A_nullCount = 0,
C_minValue = TestDataSkippingUtils.NyFallBackEdt0130,
C_maxValue = TestDataSkippingUtils.NyFallBackEst0110,
C_nullCount = 0))
@@ -160,6 +164,22 @@ class TestDataSkippingUtils extends
HoodieSparkClientTestBase with SparkAdapterS
assertEquals(Seq("file_1"), applyFilterExpr(ntzFilterExpr, input))
}
+
+ // date_format and from_unixtime render the same local time, so a
pattern with the time of day
+ // re-orders the same way
+ assertEquals(Seq("file_1"), applyFilterExpr(
+ resolveExpr(spark, "date_format(C, 'yyyy-MM-dd HH:mm') > '2026-11-01
01:20'", sourceTableSchema), input))
+ assertEquals(Seq("file_1"), applyFilterExpr(
+ resolveExpr(spark, "from_unixtime(A, 'yyyy-MM-dd HH:mm') > '2026-11-01
01:20'", sourceTableSchema), input))
+
+ // A date-only pattern renders the local date, which never runs
backwards in New York, so it still
+ // translates and prunes
+ val earlierDay = IndexRow("file_2", valueCount = 2,
+ C_minValue = new Timestamp(1793361600000L), // 2026-10-30 12:00Z
+ C_maxValue = new Timestamp(1793365200000L), // 2026-10-30 13:00Z
+ C_nullCount = 0)
+ assertEquals(Seq("file_1"), applyFilterExpr(
+ resolveExpr(spark, "date_format(C, 'yyyy-MM-dd') = '2026-11-01'",
sourceTableSchema), input :+ earlierDay))
} finally {
spark.sqlContext.setConf(SESSION_LOCAL_TIMEZONE.key, "UTC")
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
index 22f9f52dbcdd..db2d01a9b611 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieCatalystExpressionUtils.scala
@@ -184,6 +184,33 @@ class TestHoodieCatalystExpressionUtils extends
SparkAdapterSupport {
assertEquals(None, matched(FromUnixTime(longAttr, Literal("dd-MM-yyyy"),
Some("UTC"))))
}
+ @Test
+ def testTimeOfDayPatternsRequireFixedOffsetSessionZone(): Unit = {
+ val newYork = Some("America/New_York")
+ // Every allowlisted time-of-day pattern renders the local time, which
runs backwards across a DST fall-back of
+ // the session zone, and keeps the ordering in a session zone whose offset
never changes.
+ Seq("yyyy-MM-dd HH", "yyyy-MM-dd HH:mm", "yyyy-MM-dd HH:mm:ss",
"yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd'T'HH:mm:ss",
+ "yyyy-MM-dd'T'HH:mm:ss.SSS", "yyyyMMddHHmmss", "yyyy/MM/dd
HH:mm:ss").foreach { pattern =>
+ assertEquals(None, matched(DateFormatClass(tsAttr, Literal(pattern),
newYork)), pattern)
+ assertEquals(None, matched(FromUnixTime(longAttr, Literal(pattern),
newYork)), pattern)
+ assertEquals(Some(tsAttr), matched(DateFormatClass(tsAttr,
Literal(pattern), Some("UTC"))), pattern)
+ assertEquals(Some(longAttr), matched(FromUnixTime(longAttr,
Literal(pattern), Some("+08:00"))), pattern)
+ }
+ // Every allowlisted date-only pattern renders the local date, which never
runs backwards in New York.
+ Seq("yyyy", "yyyy-MM", "yyyy-MM-dd", "yyyyMMdd", "yyyy/MM/dd").foreach {
pattern =>
+ assertEquals(Some(tsAttr), matched(DateFormatClass(tsAttr,
Literal(pattern), newYork)), pattern)
+ assertEquals(Some(longAttr), matched(FromUnixTime(longAttr,
Literal(pattern), newYork)), pattern)
+ }
+ // PST resolves to America/Los_Angeles; Kolkata has no DST but historical
offset changes.
+ assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd
HH:mm"), Some("PST"))))
+ assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd
HH:mm"), Some("Asia/Kolkata"))))
+ // A non-literal or null pattern, an unknown zone or an unresolved session
zone is not matched.
+ assertEquals(None, matched(DateFormatClass(tsAttr, strAttr, newYork)))
+ assertEquals(None, matched(DateFormatClass(tsAttr, Literal(null,
StringType), newYork)))
+ assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd
HH:mm"), Some("Not/AZone"))))
+ assertEquals(None, matched(DateFormatClass(tsAttr, Literal("yyyy-MM-dd
HH:mm"), None)))
+ }
+
@Test
def testUpCastPreservesOrderingButNumericToStringDoesNot(): Unit = {
// Widening a numeric column preserves ordering, so the source attribute
is recovered.
