wombatu-kun commented on code in PR #20759:
URL: https://github.com/apache/hudi/pull/20759#discussion_r4652221437
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ExpressionIndexSupport.scala:
##########
@@ -409,23 +410,41 @@ class ExpressionIndexSupport(spark: SparkSession,
/**
* Bloom filter keys: the values of `f(col) = v` and `f(col) IN (...)`, as
strings like the writer added them. A file
* passes if it may hold any key, which is conservative across conjuncts. A
conjunct with a value that is not a
- * non-null literal gives no keys, and is skipped. So does a term whose
values the writer and the literal print
- * differently, for ex a DATE is `2023-01-02` to the writer and `19359` here
(#20567).
+ * non-null literal gives no keys, and is skipped. So does a term whose keys
cannot be rebuilt here, see
+ * [[hasWriterKeys]].
*/
private def bloomFilterKeys(use: IndexUse): List[String] =
use.conjuncts.flatMap(Leaf.parse).filter(leaf =>
use.indexed.matches(leaf.term) && hasWriterKeys(leaf.term.dataType)).flatMap {
leaf =>
val values: Seq[Any] = leaf.values.map {
case Literal(value, _) => value
case _ => null
}
- if (values.isEmpty || values.contains(null)) Seq.empty else
values.map(_.toString)
+ if (values.isEmpty || values.contains(null)) Seq.empty else
values.flatMap(writerKeys(_, leaf.term.dataType))
}.toList
+ /**
+ * Whether the writer's keys, `toString` of the external value, can be
rebuilt from a literal of this type.
+ *
+ * NOTE: Not TIMESTAMP: the writer prints a `java.sql.Timestamp` in the
executor's default time zone, or an `Instant`
+ * with `spark.sql.datetime.java8API.enabled`, which cannot be rebuilt here.
+ * NOTE: Not FLOAT/DOUBLE: `-0.0` and `0.0` print apart though they compare
equal. Not DECIMAL: it prints its scale.
+ */
private def hasWriterKeys(dataType: DataType): Boolean = dataType match {
- case _: StringType | ByteType | ShortType | IntegerType | LongType => true
+ case _: StringType | ByteType | ShortType | IntegerType | LongType |
DateType => true
case _ => false
}
+ /**
+ * The writer's keys for an internal literal value. A DATE is epoch days
here, and the writer prints a `java.sql.Date`,
+ * or a `LocalDate` with `spark.sql.datetime.java8API.enabled`; both are
probed, as they differ for edge years.
+ */
+ private def writerKeys(value: Any, dataType: DataType): Seq[String] =
dataType match {
+ case DateType =>
+ val days = value.asInstanceOf[Int]
+ Seq(DateTimeUtils.toJavaDate(days).toString,
DateTimeUtils.daysToLocalDate(days).toString).distinct
Review Comment:
`toJavaDate(days)` uses the reader's default zone, while the legacy writer
key uses the executor's zone; for `2011-12-30`, a Pacific/Apia writer stores
`2011-12-31` but a UTC reader probes only `2011-12-30`. Please guard legacy
DATE pruning on a known writer zone, with a conservative fallback for existing
indexes, and cover this cross-zone case.
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestExpressionIndex.scala:
##########
@@ -1934,14 +1934,25 @@ class TestExpressionIndex extends
HoodieSparkSqlTestBase with SparkAdapterSuppor
spark.sql(s"INSERT INTO $tableName VALUES ('2023-09-23
09:42:17','u1','rider-F',34.15,'sao_paulo')")
spark.sql(s"INSERT INTO $tableName VALUES ('2023-09-24
09:00:00','u2','rider-Z',12.50,'sao_paulo')")
spark.sql(s"CREATE INDEX ts_year ON $tableName USING column_stats(ts)
options(expr='date_format', format='yyyy')")
- spark.sql(s"CREATE INDEX ts_two_days ON $tableName USING
bloom_filters(ts) options(expr='date_add', days='2')")
withSQLConf(DataSourceReadOptions.ENABLE_DATA_SKIPPING.key -> "true",
HoodieMetadataConfig.ENABLE.key -> "true",
"spark.sql.adaptive.enabled" -> "false") {
// Spark casts the indexed string to INT here, which no rule ranges:
the index cannot prune this filter
assertScan(tableName, "date_format(ts, 'yyyy') = 2023 AND fare =
12.5", Seq("rider-Z"), expectedFiles = 1)
- // A bloom filter holds a DATE as the writer's string, not as the
epoch days of the literal: it cannot answer this
- assertScan(tableName, "date_add(ts, 2) IN (DATE'2023-09-25',
DATE'2023-09-27')", Seq("rider-F"), expectedFiles = 2)
+ }
+
+ // A bloom filter holds a DATE as the writer's string, a java.sql.Date
or, with the java8 API, a LocalDate,
+ // not as the epoch days of the literal (#20567)
+ Seq("false", "true").foreach { java8Api =>
+ withSQLConf("spark.sql.datetime.java8API.enabled" -> java8Api) {
+ spark.sql(s"CREATE INDEX ts_two_days ON $tableName USING
bloom_filters(ts) options(expr='date_add', days='2')")
+ }
+ withSQLConf(DataSourceReadOptions.ENABLE_DATA_SKIPPING.key ->
"true", HoodieMetadataConfig.ENABLE.key -> "true",
+ "spark.sql.adaptive.enabled" -> "false") {
+ assertScan(tableName, "date_add(ts, 2) = DATE'2023-09-25'",
Seq("rider-F"), expectedFiles = 1)
Review Comment:
These 2023 dates produce the same `java.sql.Date` and `LocalDate` key, so
both writer-setting iterations would still pass if either conversion were
removed. Add a year-zero case under both settings to exercise the distinct keys.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]