Github user xuanyuanking commented on a diff in the pull request:
https://github.com/apache/spark/pull/21502#discussion_r193976536
--- Diff:
sql/core/src/test/scala/org/apache/spark/sql/execution/joins/BroadcastJoinSuite.scala
---
@@ -153,6 +154,23 @@ class BroadcastJoinSuite extends QueryTest with
SQLTestUtils {
}
test("SPARK-22575: remove allocated blocks when they are not needed
anymore") {
+ val blockManager = sparkContext.env.blockManager
+ def broadcastedBlockIds: Seq[BlockId] = {
+ blockManager.getMatchingBlockIds(blockId => {
+ blockId.isBroadcast &&
blockManager.getStatus(blockId).get.storageLevel.deserialized
+ }).distinct
+ }
+ def isHashedRelationPresent(blockIds: Seq[BlockId]): Boolean = {
+ val blockValues = blockIds.flatMap { id =>
+ val block = blockManager.getSingle[Any](id)
+ if (block.isDefined) {
--- End diff --
Maybe we should add more comments here, is this the root cause about last
failure?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
