cloud-fan commented on code in PR #52646:
URL: https://github.com/apache/spark/pull/52646#discussion_r2612916272
##########
core/src/main/scala/org/apache/spark/storage/BlockInfoManager.scala:
##########
@@ -573,11 +600,75 @@ private[storage] class
BlockInfoManager(trackingCacheVisibility: Boolean = false
}
}
blockInfoWrappers.clear()
+ rddToBlockIds.clear()
+ broadcastToBlockIds.clear()
+ sessionToBlockIds.clear()
readLocksByTask.clear()
writeLocksByTask.clear()
invisibleRDDBlocks.synchronized {
invisibleRDDBlocks.clear()
}
}
+ /**
+ * Return all blocks in the cache mapping for a given key.
+ */
+ private def getBlockIdsFromMapping[K](
+ map: ConcurrentHashMap[K, ConcurrentHashMap.KeySetView[BlockId,
java.lang.Boolean]],
+ key: K): Seq[BlockId] = {
+ Option(map.get(key)).map(_.asScala.toSeq).getOrElse(Seq.empty)
+ }
+
+ /**
+ * Add a block ID to the corresponding cache mapping based on its type.
+ */
+ private def addToMapping(blockId: BlockId): Unit = {
+ blockId match {
+ case rddBlockId: RDDBlockId =>
+ rddToBlockIds
+ .computeIfAbsent(rddBlockId.rddId, _ =>
ConcurrentHashMap.newKeySet())
+ .add(blockId)
+ case broadcastBlockId: BroadcastBlockId =>
+ broadcastToBlockIds
+ .computeIfAbsent(broadcastBlockId.broadcastId, _ =>
ConcurrentHashMap.newKeySet())
+ .add(blockId)
+ case cacheId: CacheId =>
+ sessionToBlockIds
+ .computeIfAbsent(cacheId.sessionUUID, _ =>
ConcurrentHashMap.newKeySet())
+ .add(blockId)
+ case _ => // Do nothing for other block types
+ }
+ }
+
+ /**
+ * Remove a block ID from the corresponding cache mapping based on its type.
+ */
+ private def removeFromMapping(blockId: BlockId): Unit = {
+ def doRemove[K](
+ map: ConcurrentHashMap[K, ConcurrentHashMap.KeySetView[BlockId,
java.lang.Boolean]],
+ key: K,
+ block: BlockId): Unit = {
+ map.compute(key,
+ (_, set) => {
+ if (null != set) {
+ set.remove(block)
+ if (set.isEmpty) null else set
Review Comment:
does setting value to null remove the entry from the map?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
