vinodkc commented on code in PR #53312:
URL: https://github.com/apache/spark/pull/53312#discussion_r3488274515
##########
sql/core/src/test/scala/org/apache/spark/sql/StatisticsCollectionSuite.scala:
##########
@@ -598,6 +598,127 @@ class StatisticsCollectionSuite extends
StatisticsCollectionTestBase with Shared
}
}
+ test("TimeType column statistics collection and precisions") {
+ val table = "time_stats_table"
+ withTable(table) {
+ // Test all precisions (0-6) with NULL handling
+ sql(s"""
+ CREATE TABLE $table (
+ id INT,
+ time_p0 TIME(0),
+ time_p1 TIME(1),
+ time_p2 TIME(2),
+ time_p3 TIME(3),
+ time_p4 TIME(4),
+ time_p5 TIME(5),
+ time_p6 TIME(6)
+ ) USING parquet
+ """)
+
+ sql(s"""
+ INSERT INTO $table VALUES
+ (1, TIME '08:30:00', TIME '08:30:00.1', TIME '08:30:00.12',
+ TIME '08:30:00.123', TIME '08:30:00.1234', TIME '08:30:00.12345',
+ TIME '08:30:00.123456'),
+ (2, TIME '17:45:30', TIME '17:45:30.9', TIME '17:45:30.98',
+ TIME '17:45:30.987', TIME '17:45:30.9876', TIME '17:45:30.98765',
+ TIME '17:45:30.987654'),
+ (3, TIME '12:00:00', TIME '12:00:00.5', TIME '12:00:00.5',
+ TIME '12:00:00.5', TIME '12:00:00.5', TIME '12:00:00.5',
+ TIME '12:00:00.5'),
+ (4, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
+ """)
+
+ sql(s"""ANALYZE TABLE $table COMPUTE STATISTICS FOR COLUMNS
+ time_p0, time_p1, time_p2, time_p3, time_p4, time_p5, time_p6""")
+
+ val catalogTable = getCatalogTable(table)
+
+ for (precision <- 0 to 6) {
+ val col = s"time_p$precision"
+ val stats = catalogTable.stats.get.colStats(col)
+
+ // Verify basic statistics
+ assert(stats.distinctCount.isDefined, s"Distinct count should be
defined for $col")
+ assert(stats.min.isDefined, s"Min should be defined for $col")
+ assert(stats.max.isDefined, s"Max should be defined for $col")
+ assert(stats.nullCount == Some(1), s"Null count should be 1 for $col")
+ assert(stats.avgLen == Some(8), s"Avg length should be 8 bytes for
$col")
+ assert(stats.maxLen == Some(8), s"Max length should be 8 bytes for
$col")
+
+ // Verify format for each precision
+ val minStr = stats.min.get.asInstanceOf[String]
+ val maxStr = stats.max.get.asInstanceOf[String]
+ assert(minStr.matches("\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?"),
+ s"Min should be time format for $col, got: $minStr")
Review Comment:
Fixed, added `expectedMins`/`expectedMaxs` maps with concrete values per
precision (e.g. `"08:30:00.123"` for TIME(3)) and assert exact equality, so a
swapped or wrong min/max will now fail the test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
