mridulm commented on code in PR #39459:
URL: https://github.com/apache/spark/pull/39459#discussion_r1119073044
##########
core/src/test/scala/org/apache/spark/storage/BlockManagerSuite.scala:
##########
@@ -2266,6 +2270,160 @@ class BlockManagerSuite extends SparkFunSuite with
Matchers with PrivateMethodTe
}
}
+ test("SPARK-41497: getOrElseUpdateRDDBlock do compute based on cache
visibility statue") {
+ val store = makeBlockManager(8000, "executor1")
+ val blockId = RDDBlockId(rddId = 1, splitIndex = 1)
+ var computed: Boolean = false
+ val data = Seq(1, 2, 3)
+ val makeIterator = () => {
+ computed = true
+ data.iterator
+ }
+
+ // Cache doesn't exist and is not visible.
+ assert(store.getStatus(blockId).isEmpty &&
!store.isRDDBlockVisible(blockId))
+ val res1 = store.getOrElseUpdateRDDBlock(
+ 1, blockId, StorageLevel.MEMORY_ONLY, classTag[Int], makeIterator)
+ // Put cache successfully and reported block task info.
+ assert(res1.isLeft && computed)
+ verify(master, times(1)).updateRDDBlockTaskInfo(blockId, 1)
+
+ // Cache exists but not visible.
+ computed = false
+ assert(store.getStatus(blockId).nonEmpty &&
!store.isRDDBlockVisible(blockId))
+ val res2 = store.getOrElseUpdateRDDBlock(
+ 1, blockId, StorageLevel.MEMORY_ONLY, classTag[Int], makeIterator)
+ // Load cache successfully and reported block task info.
+ assert(res2.isLeft && computed)
+ verify(master, times(2)).updateRDDBlockTaskInfo(blockId, 1)
Review Comment:
That makes sense, thanks for clarifying !
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
