philo-he commented on code in PR #13531:
URL: https://github.com/apache/gluten/pull/13531#discussion_r4557989296
##########
backends-velox/src/test/scala/org/apache/gluten/sql/GlutenBloomFilterFallbackSuite.scala:
##########
@@ -486,4 +487,236 @@ class GlutenBloomFilterFallbackSuite extends
WholeStageTransformerSuite {
}
}
}
+
+ // Gated to Spark 3.5+: Spark's BloomFilterAggregate only supports
StringType starting in
+ // Spark 3.5 (SPARK-43726); on Spark 3.4 CheckAnalysis rejects
bloom_filter_agg(string) with
+ // DATATYPE_MISMATCH.BLOOM_FILTER_WRONG_TYPE before Optimizer/Gluten rules
run.
+ testWithMinSparkVersion(
+ "GLUTEN-13489: bloom_filter_agg on string column falls back to vanilla
Spark",
+ "3.5") {
+ if
(BackendsApiManager.getSettings.requireBloomFilterAggMightContainJointFallback())
{
+ val table = "bloom_filter_str_test"
+ withTempView(table) {
+ Seq("a", "b", "c").toDF("s").createOrReplaceTempView(table)
+ val aggDf = spark.sql(s"SELECT bloom_filter_agg(s) AS bf FROM $table")
+ val bytes = aggDf.collect().head.getAs[Array[Byte]]("bf")
+
assert(!aggDf.queryExecution.executedPlan.toString.contains("velox_bloom_filter_agg"))
+ val bf = BloomFilter.readFrom(new ByteArrayInputStream(bytes))
+ assert(bf.mightContainString("a"))
+ assert(!bf.mightContainString("z"))
+
+ val df = spark.sql(
+ s"""
+ |SELECT might_contain((SELECT bloom_filter_agg(s) FROM $table),
xxhash64(s))
+ |FROM $table
+ |""".stripMargin)
+ checkAnswer(df, Seq(Row(false), Row(false), Row(false)))
+ val plan = df.queryExecution.executedPlan.toString
+ assert(!plan.contains("velox_might_contain"))
+ assert(!plan.contains("velox_bloom_filter_agg"))
+ }
+ }
+ }
+
+ test("GLUTEN-13489: bloom_filter_agg on nullable column filters out nulls
and runs natively") {
+ if
(BackendsApiManager.getSettings.requireBloomFilterAggMightContainJointFallback())
{
+ val table = "bloom_filter_nullable_test"
+ withTempView(table) {
+ Seq[java.lang.Long](1L, null, 2L,
3L).toDF("col").createOrReplaceTempView(table)
+ val df = spark.sql(
+ s"""
+ |SELECT col
+ |FROM $table
+ |WHERE might_contain((SELECT bloom_filter_agg(col) FROM $table),
col)
+ |""".stripMargin)
+ checkAnswer(df, Seq(Row(1L), Row(2L), Row(3L)))
+ val nativeAggs = collectWithSubqueries(df.queryExecution.executedPlan)
{
+ case h: HashAggregateExecBaseTransformer => h
+ }
+ assert(nativeAggs.size == 2, df.queryExecution.executedPlan.toString)
+
+ val filteredDf = spark.sql(
+ s"""
+ |SELECT col
+ |FROM $table
+ |WHERE might_contain(
+ | (SELECT bloom_filter_agg(col) FILTER (WHERE col IS NULL OR col
> 1L) FROM $table),
+ | col)
+ |""".stripMargin)
+ checkAnswer(filteredDf, Seq(Row(2L), Row(3L)))
+ val filteredNativeAggs =
collectWithSubqueries(filteredDf.queryExecution.executedPlan) {
+ case h: HashAggregateExecBaseTransformer => h
+ }
+ assert(filteredNativeAggs.size == 2,
filteredDf.queryExecution.executedPlan.toString)
+ }
+
+ val allNullTable = "bloom_filter_all_null_test"
+ withTempView(allNullTable) {
+ Seq[java.lang.Long](null,
null).toDF("col").createOrReplaceTempView(allNullTable)
+ val df = spark.sql(
+ s"SELECT might_contain((SELECT bloom_filter_agg(col) FROM
$allNullTable), 1L)")
+ checkAnswer(df, Seq(Row(null)))
Review Comment:
This test only covers the native path. When `VeloxBloomFilterAggregate` runs
in the JVM instead (for example with hash-agg offload disabled, or after a
whole-stage fallback), I think this query returns `false` instead of `NULL`.
Why: when no value is added, Spark's `BloomFilterAggregate.eval` returns
`null`, but `VeloxBloomFilterAggregate.eval` always returns the serialized
filter. Here the new `IS NOT NULL` filter drops every row, so the JVM path
returns an empty filter, and `might_contain` on an empty filter returns `false`.
Could you add a case with `GlutenConfig.COLUMNAR_HASHAGG_ENABLED.key ->
"false"`? If it returns `false`, `VeloxBloomFilterAggregate.eval` should return
`null` when no value was added, as Spark does.
##########
backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala:
##########
@@ -49,12 +54,12 @@ case class BloomFilterMightContainJointRewriteRule(
Expression,
Int,
Int) => TypedImperativeAggregate[T]): Expression = expr match {
- case BloomFilterAggregate(
+ case bf @ BloomFilterAggregate(
child,
estimatedNumItemsExpression,
numBitsExpression,
mutableAggBufferOffset,
- inputAggBufferOffset) =>
+ inputAggBufferOffset) if !isKeepVanilla(bf) =>
Review Comment:
This rule now depends on the `KEEP_VANILLA` tag set by the Logical rule: any
bloom filter expression without the tag is still rewritten. `transform` and
`withNewChildren` keep tags, but a rule that rebuilds the expression with
`copy(...)` or its constructor drops them. I didn't find such a rule in Spark
today. If one appears later, the #13489 failures come back silently; for
example, a standalone `bloom_filter_agg` would return Velox-format bytes again.
Could you make this rule safe even when a tag is missing?
`InjectRuntimeFilter` always wraps both inputs in `XxHash64`:
`bloom_filter_agg(xxhash64(key))` and `might_contain(..., xxhash64(key))`. If
this rule only rewrote expressions of that shape, user-written calls would stay
on vanilla Spark even if their tag were lost.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]