uros-b commented on code in PR #56101: URL: https://github.com/apache/spark/pull/56101#discussion_r3683524797
########## sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNearestByJoinExec.scala: ########## @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.joins + +import java.util.{Comparator, PriorityQueue => JPriorityQueue} + +import org.apache.spark.SparkException +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.plans.{InnerLike, JoinType, LeftOuter, NearestByDirection, NearestByDistance} +import org.apache.spark.sql.catalyst.plans.physical._ +import org.apache.spark.sql.catalyst.util.TypeUtils +import org.apache.spark.sql.execution.{ExplainUtils, SparkPlan} +import org.apache.spark.sql.execution.metric.SQLMetrics + +/** + * Heap entry storing an index into the broadcast array alongside its ranking value. + * Using a case class with primitive `Int` field avoids boxing that `(Int, Any)` tuples incur. + */ +private[joins] case class HeapEntry(index: Int, rankingValue: Any) + +/** + * Physical operator for NearestByJoin that avoids materializing the full cross product. + * For each left row, iterates all broadcast right rows maintaining a bounded priority + * queue of size k, then emits the top-k matches directly. + * + * The right side is fully broadcast to all partitions. This operator only fires when + * the right side fits within the broadcast join threshold + * ([[org.apache.spark.sql.internal.SQLConf.autoBroadcastJoinThreshold]]). For right tables + * exceeding this threshold, the existing cross-product + aggregate rewrite is used as + * fallback. Tie-breaking among equal ranking values is non-deterministic (matches the + * existing rewrite behavior). + */ +case class BroadcastNearestByJoinExec( + left: SparkPlan, + right: SparkPlan, + joinType: JoinType, + numResults: Int, + rankingExpression: Expression, + direction: NearestByDirection) extends BaseJoinExec { + + override def condition: Option[Expression] = None + override def leftKeys: Seq[Expression] = Seq.empty + override def rightKeys: Seq[Expression] = Seq.empty + + override def simpleStringWithNodeId(): String = { Review Comment: EXPLAIN FORMATTED never shows the ranking expression. simpleStringWithNodeId is overridden to "$nodeName $joinType k=$numResults $direction", and verboseStringWithOperatorId is inherited from BaseJoinExec, which takes the leftKeys.isEmpty && rightKeys.isEmpty branch and prints only Join type and Join condition: None. So the one expression that determines which rows come out is absent from both the tree line and the details block; you cannot tell two different NEAREST BY queries apart in a formatted plan. Every other join operator surfaces its defining predicate there. Overriding verboseStringWithOperatorId to emit the ranking expression, k, and direction would fix it, and it is worth one assertion in the suite since nothing currently checks explain output. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
