rdtr commented on code in PR #13531:
URL: https://github.com/apache/gluten/pull/13531#discussion_r4572676849


##########
backends-velox/src/main/scala/org/apache/gluten/extension/BloomFilterMightContainJointRewriteRule.scala:
##########
@@ -81,3 +87,113 @@ case class BloomFilterMightContainJointRewriteRule(
     }
   }
 }
+
+object BloomFilterMightContainJointRewriteRule {
+  private val KEEP_VANILLA_BLOOM_FILTER_TAG =
+    TreeNodeTag[Unit]("org.apache.gluten.keepVanillaBloomFilter")
+
+  private def tagKeepVanilla(expr: Expression): Expression = {
+    expr.setTagValue(KEEP_VANILLA_BLOOM_FILTER_TAG, ())
+    expr
+  }
+
+  private def isKeepVanilla(expr: Expression): Boolean = {
+    expr.getTagValue(KEEP_VANILLA_BLOOM_FILTER_TAG).isDefined
+  }
+
+  class Logical(spark: SparkSession) extends Rule[LogicalPlan] {
+    override def apply(plan: LogicalPlan): LogicalPlan = {
+      if (!GlutenConfig.get.enableNativeBloomFilter) {
+        return plan
+      }
+      rewritePlan(plan)
+    }
+
+    private def rewritePlan(plan: LogicalPlan): LogicalPlan = {
+      plan.mapChildren(rewritePlan).mapExpressions(rewriteExpr)
+    }
+
+    private def rewriteExpr(expr: Expression): Expression = {
+      val rewritten = expr.mapChildren(rewriteExpr) match {
+        case sub: SubqueryExpression =>
+          sub.withNewPlan(rewritePlan(sub.plan))
+        case other =>
+          other
+      }
+      rewritten match {
+        case mc: BloomFilterMightContain =>
+          rewriteMightContain(mc)
+        case bfAgg: BloomFilterAggregate =>
+          tagKeepVanilla(bfAgg)
+        case other =>
+          other
+      }
+    }
+
+    private def rewriteMightContain(mc: BloomFilterMightContain): Expression = 
{
+      mc.left match {
+        case sub: ScalarSubquery =>
+          rewritePairedSubquery(sub.plan)
+            .map(
+              rewrittenPlan =>
+                VeloxBloomFilterMightContain(sub.withNewPlan(rewrittenPlan), 
mc.right))
+            .getOrElse(tagKeepVanilla(mc))
+        case _ =>

Review Comment:
   A foldable first argument always stays vanilla here. On main every 
`might_contain` was rewritten, so a Velox-format constant used to work. With 
this PR, Spark's `might_contain` reads it and fails (Spark 4.0, same on 3.5):
   
   ```
   java.io.IOException: Unexpected Bloom filter version number (25165824)
     at 
org.apache.spark.util.sketch.BloomFilterImpl.readFrom0(BloomFilterImpl.java:251)
     ...
     at 
org.apache.spark.sql.catalyst.expressions.BloomFilterMightContain.deserialize(BloomFilterMightContain.scala:122)
   ```
   
   Repro: build the bytes with `VeloxBloomFilterAggregate(col, lit(1000L), 
lit(8192L), 0, 0)` registered as a function, then run `SELECT col FROM t WHERE 
might_contain(unhex('<hex>'), col)`. On main, a standalone 
`bloom_filter_agg(long_col)` returned exactly these bytes (item 4 in #13489), 
so users may have stored some.
   
   The first byte tells the two formats apart. Velox writes its version as one 
byte (`kBloomFilterV1 = 1` in velox/common/base/BloomFilter.h). Spark writes a 
big-endian int version, so its first byte is 0. My branch sends a foldable 
`BinaryType` argument whose first byte is 1 to `VeloxBloomFilterMightContain`: 
https://github.com/rdtr/incubator-gluten/blob/67ed8efa8d05df569194a2a2e9b9b4ff644f3d82/backends-velox/src/main/scala/org/apache/gluten/extension/RewriteBloomFilterPairs.scala#L168-L175
   
   I don't think it's a blocker if stored Velox bytes seem too rare to care 
about. In that case, could you add a test that pins the current behavior?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to