wbo4958 commented on code in PR #43494:
URL: https://github.com/apache/spark/pull/43494#discussion_r1375821367
##########
core/src/main/scala/org/apache/spark/resource/ResourceAllocator.scala:
##########
@@ -29,59 +65,45 @@ private[spark] trait ResourceAllocator {
protected def resourceName: String
protected def resourceAddresses: Seq[String]
- protected def slotsPerAddress: Int
/**
- * Map from an address to its availability, a value > 0 means the address is
available,
- * while value of 0 means the address is fully assigned.
- *
- * For task resources ([[org.apache.spark.scheduler.ExecutorResourceInfo]]),
this value
- * can be a multiple, such that each address can be allocated up to
[[slotsPerAddress]]
- * times.
+ * Map from an address to its availability default to RESOURCE_TOTAL_AMOUNT,
a value > 0 means
+ * the address is available, while value of 0 means the address is fully
assigned.
*/
- private lazy val addressAvailabilityMap = {
- mutable.HashMap(resourceAddresses.map(_ -> slotsPerAddress): _*)
+ protected lazy val addressAvailabilityMap = {
+ mutable.HashMap(resourceAddresses.map(address => address ->
RESOURCE_TOTAL_AMOUNT): _*)
}
/**
- * Sequence of currently available resource addresses.
- *
- * With [[slotsPerAddress]] greater than 1, [[availableAddrs]] can contain
duplicate addresses
- * e.g. with [[slotsPerAddress]] == 2, availableAddrs for addresses 0 and 1
can look like
- * Seq("0", "0", "1"), where address 0 has two assignments available, and 1
has one.
+ * Sequence of currently available resource addresses which is not fully
assigned.
*/
def availableAddrs: Seq[String] = addressAvailabilityMap
- .flatMap { case (addr, available) =>
- (0 until available).map(_ => addr)
- }.toSeq.sorted
+ .filter(addresses => addresses._2 > 0).keys.toSeq.sorted
/**
* Sequence of currently assigned resource addresses.
Review Comment:
Yeah, You're right. I haven't considered the UI case. Yes, the UI won't have
this detail yet.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
