This is an automated email from the ASF dual-hosted git repository.
peter-toth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 84db3edbee2a [SPARK-60702][SQL][FOLLOWUP] Expect no bloom filter from
a non-deterministic creation side without DPP
84db3edbee2a is described below
commit 84db3edbee2abc76d2892c3cc980bbdaaf954aa9
Author: Peter Toth <[email protected]>
AuthorDate: Sat Nov 7 19:16:00 2026 +0100
[SPARK-60702][SQL][FOLLOWUP] Expect no bloom filter from a
non-deterministic creation side without DPP
### What changes were proposed in this pull request?
This fixes two tests of #59916 that fail on master. Both check with DPP off
that master still adds the bloom filter from a non-deterministic creation side.
SPARK-60808 (#60176), merged before #59916, keeps that bloom filter out now, so
the tests now expect none:
- "SPARK-60702: do not add bloom filter if the creation side that the
subquery reads is not deterministic";
- "SPARK-60702: do not add bloom filter whose creation side is a
non-deterministic pruned side".
### Why are the changes needed?
`InjectRuntimeFilterSuite` fails on master. #59916 was tested on a base
without SPARK-60808.
### Does this PR introduce _any_ user-facing change?
No. Test only.
### How was this patch tested?
`InjectRuntimeFilterSuite` passes, 62 of 62. On master `da165c1c658` the
two tests fail with `List() had size 0 instead of expected size 1`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code with Claude Opus 5.5
Closes #60206 from peter-toth/SPARK-60702-followup-tests-after-60808.
Authored-by: Peter Toth <[email protected]>
Signed-off-by: Peter Toth <[email protected]>
skip-checks: true
---
.../scala/org/apache/spark/sql/InjectRuntimeFilterSuite.scala | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/InjectRuntimeFilterSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/InjectRuntimeFilterSuite.scala
index 07c6e66d8c6c..4e5541adaf1c 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/InjectRuntimeFilterSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/InjectRuntimeFilterSuite.scala
@@ -1352,11 +1352,11 @@ class InjectRuntimeFilterSuite extends
SharedSparkSession
"deterministic") {
// The build plan of the DPP filter would be the creation side of the
bloom filter. A bloom
// filter would evaluate it again, apart from the join, and could drop
rows (SPARK-60808).
- // Cleanup drops such a DPP filter.
+ // Cleanup drops such a DPP filter. Without DPP, SPARK-60808 keeps the
bloom filter out too.
val query = "select * from bf5part join (select * from bf2 where a2 = 36
and " +
"cast(reflect('java.lang.Math', 'random') as double) < 2.0) d on
bf5part.f5 = d.c2"
withDppTestConf() {
- assertBloomFilterWithoutDpp(query)
+ assertBloomFilterWithoutDpp(query, 0)
assert(bloomFilterKeys(query).isEmpty)
assert(dynamicPruningFilters(query).isEmpty)
}
@@ -1366,7 +1366,7 @@ class InjectRuntimeFilterSuite extends SharedSparkSession
"side") {
// Nothing broadcasts `f`, the deterministic build plan of the DPP filter
on `d.k`, so that
// filter is dropped. `d` is too small to take a bloom filter, so `f`
would take one, built
- // from `d`, which is not deterministic (SPARK-60808).
+ // from `d`, which is not deterministic (SPARK-60808). Without DPP,
SPARK-60808 keeps it out.
withTable("ndd", "ndf") {
spark.range(40).selectExpr("id as k", "id as
w").write.partitionBy("k").saveAsTable("ndd")
spark.range(40000).selectExpr("id % 40 as k", "id as
v").write.saveAsTable("ndf")
@@ -1380,7 +1380,7 @@ class InjectRuntimeFilterSuite extends SharedSparkSession
SQLConf.RUNTIME_BLOOM_FILTER_APPLICATION_SIDE_SCAN_SIZE_THRESHOLD.key ->
(nddSize + 1).toString,
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
- assertBloomFilterWithoutDpp(query)
+ assertBloomFilterWithoutDpp(query, 0)
assert(bloomFilterKeys(query).isEmpty)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]