attilapiros commented on a change in pull request #24499: [SPARK-25888][Core]
Serve local disk persisted blocks by the external service after releasing
executor by dynamic allocation
URL: https://github.com/apache/spark/pull/24499#discussion_r279928981
##########
File path:
core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
##########
@@ -496,19 +530,25 @@ private[spark] class BlockManagerInfo(
timeMs: Long,
val maxOnHeapMem: Long,
val maxOffHeapMem: Long,
- val slaveEndpoint: RpcEndpointRef)
+ val slaveEndpoint: RpcEndpointRef,
+ val externalShuffleServiceBlockStatus: Option[JHashMap[BlockId,
BlockStatus]])
extends Logging {
val maxMem = maxOnHeapMem + maxOffHeapMem
+ val externalShuffleServiceEnabled =
externalShuffleServiceBlockStatus.isDefined
+
private var _lastSeenMs: Long = timeMs
private var _remainingMem: Long = maxMem
// Mapping from block id to its status.
private val _blocks = new JHashMap[BlockId, BlockStatus]
- // Cached blocks held by this BlockManager. This does not include broadcast
blocks.
- private val _cachedBlocks = new mutable.HashSet[BlockId]
+ /**
+ * Cached blocks held exclusively by this BlockManager. This does not
include broadcast blocks
+ * and local disc persisted blocks when external shuffle service is enabled.
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
