liujiayi771 commented on code in PR #8931:
URL: https://github.com/apache/incubator-gluten/pull/8931#discussion_r2902864461
##########
backends-velox/src/main/scala/org/apache/gluten/execution/HashJoinExecTransformer.scala:
##########
@@ -125,9 +131,52 @@ case class BroadcastHashJoinExecTransformer(
override def columnarInputRDDs: Seq[RDD[ColumnarBatch]] = {
val streamedRDD = getColumnarInputRDDs(streamedPlan)
+ val executionId =
sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY)
+ if (executionId != null) {
+ logWarning(
+ s"Trace broadcast table data $buildBroadcastTableId" + " " +
+ "and the execution id is " + executionId)
+ GlutenDriverEndpoint.collectResources(executionId, buildBroadcastTableId)
+ } else {
+ logWarning(
+ s"Can not trace broadcast table data $buildBroadcastTableId" +
+ s" because execution id is null." +
+ s" Will clean up until expire time.")
+ }
+
val broadcast = buildPlan.executeBroadcast[BuildSideRelation]()
- val broadcastRDD = VeloxBroadcastBuildSideRDD(sparkContext, broadcast)
+ val bloomFilterPushdownSize = if
(VeloxConfig.get.hashProbeDynamicFilterPushdownEnabled) {
+ VeloxConfig.get.hashProbeBloomFilterPushdownMaxSize
+ } else {
+ -1
+ }
+ val context =
+ BroadcastHashJoinContext(
+ buildKeyExprs,
+ substraitJoinType,
+ buildSide == BuildRight,
+ condition.isDefined,
+ joinType.isInstanceOf[ExistenceJoin],
+ buildPlan.output,
+ buildBroadcastTableId,
+ isNullAwareAntiJoin,
+ bloomFilterPushdownSize,
+ VeloxConfig.get.veloxBroadcastHashTableBuildThreads
+ )
+ val broadcastRDD = VeloxBroadcastBuildSideRDD(sparkContext, broadcast,
context)
// FIXME: Do we have to make build side a RDD?
streamedRDD :+ broadcastRDD
}
}
+
+case class BroadcastHashJoinContext(
+ buildSideJoinKeys: Seq[Expression],
+ substraitJoinType: JoinRel.JoinType,
+ buildRight: Boolean,
+ hasMixedFiltCondition: Boolean,
Review Comment:
Nit: Filter
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
