ivoson commented on code in PR #39459:
URL: https://github.com/apache/spark/pull/39459#discussion_r1070632750
##########
core/src/main/scala/org/apache/spark/storage/BlockManager.scala:
##########
@@ -1325,6 +1325,64 @@ private[spark] class BlockManager(
blockInfoManager.releaseAllLocksForTask(taskAttemptId)
}
+ /**
+ * Retrieve the given rdd block if it exists and is visible, otherwise call
the provided
+ * `makeIterator` method to compute the block, persist it, and return its
values.
+ *
+ * @return either a BlockResult if the block was successfully cached, or an
iterator if the block
+ * could not be cached.
+ */
+ def getOrElseUpdateRDDBlock[T](
+ taskId: Long,
+ blockId: RDDBlockId,
+ level: StorageLevel,
+ classTag: ClassTag[T],
+ makeIterator: () => Iterator[T]): Either[BlockResult, Iterator[T]] = {
+ var getIterator = makeIterator
+
+ // Attempt to read the block from local or remote storage. If it's
present, then we don't need
+ // to go through the local-get-or-put path.
+ if (master.isRDDBlockVisible(blockId)) { // Read from cache only when the
block is visible.
+ get[T](blockId)(classTag) match {
+ case Some(block) =>
+ return Left(block)
+ case _ =>
+ // Need to compute the block.
+ }
+ // Initially we hold no locks on this block.
+ } else {
+ // Need to compute the block, since the block maybe already exists, force
+ // compute the block here.
+ val iterator = makeIterator()
+ getIterator = () => iterator
+ }
+
+ doPutIterator(blockId, getIterator, level, classTag, keepReadLock = true)
match {
+ case None =>
+ // Report taskId -> blockId relationship to master.
+ master.updateRDDBlockTaskInfo(blockId, taskId)
Review Comment:
In the latest implementation, the replication logic keeps the same as before.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
