cloud-fan commented on code in PR #47533:
URL: https://github.com/apache/spark/pull/47533#discussion_r1696676905
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/exchange/BroadcastExchangeExec.scala:
##########
@@ -61,23 +61,33 @@ trait BroadcastExchangeLike extends Exchange {
*/
def relationFuture: Future[broadcast.Broadcast[Any]]
+ @transient
+ private lazy val triggerFuture: Future[Any] = {
+ SQLExecution.withThreadLocalCaptured(session,
BroadcastExchangeExec.executionContext) {
+ // Trigger broadcast preparation which can involve expensive operations
like waiting on
+ // subqueries and file listing.
+ executeQuery(null)
+ null
+ }
+ }
+
+ protected def completionFuture:
scala.concurrent.Future[broadcast.Broadcast[Any]]
+
/**
* The asynchronous job that materializes the broadcast. It's used for
registering callbacks on
* `relationFuture`. Note that calling this method may not start the
execution of broadcast job.
* It also does the preparations work, such as waiting for the subqueries.
*/
- final def submitBroadcastJob:
scala.concurrent.Future[broadcast.Broadcast[Any]] = executeQuery {
- materializationStarted.set(true)
+ final def submitBroadcastJob():
scala.concurrent.Future[broadcast.Broadcast[Any]] = {
+ triggerFuture
completionFuture
}
- protected def completionFuture:
scala.concurrent.Future[broadcast.Broadcast[Any]]
-
/**
* Cancels broadcast job with an optional reason.
*/
final def cancelBroadcastJob(reason: Option[String]): Unit = {
- if (isMaterializationStarted() && !this.relationFuture.isDone) {
+ if (!this.relationFuture.isDone) {
Review Comment:
I do not re-implement broadcast cancellation, as we need more refactoring to
move the creation of `Future` to `BroadcastExchangeLike`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
