tgravescs commented on a change in pull request #26078: [SPARK-29151][CORE]
Support fractional resources for task resource scheduling
URL: https://github.com/apache/spark/pull/26078#discussion_r337558880
##########
File path:
core/src/main/scala/org/apache/spark/resource/ResourceAllocator.scala
##########
@@ -30,27 +30,36 @@ trait ResourceAllocator {
protected def resourceName: String
protected def resourceAddresses: Seq[String]
+ protected def slotsPerAddress: Int
/**
- * Map from an address to its availability, the value `true` means the
address is available,
- * while value `false` means the address is assigned.
+ * Map from an address to its availability, a value > 0 means the address is
available,
+ * while value of 0 means the address is fully assigned.
+ *
+ * For task resources ([[org.apache.spark.scheduler.ExecutorResourceInfo]]),
this value
+ * can be a multiple, such that each address can be allocated up to
[[slotsPerAddress]]
+ * times.
+ *
* TODO Use [[OpenHashMap]] instead to gain better performance.
*/
- private lazy val addressAvailabilityMap =
mutable.HashMap(resourceAddresses.map(_ -> true): _*)
+ private lazy val addressAvailabilityMap = {
+ mutable.HashMap(resourceAddresses.map(_ -> slotsPerAddress): _*)
+ }
/**
* Sequence of currently available resource addresses.
*/
- def availableAddrs: Seq[String] = addressAvailabilityMap.flatMap { case
(addr, available) =>
- if (available) Some(addr) else None
- }.toSeq
+ def availableAddrs: Seq[String] = addressAvailabilityMap
+ .flatMap { case (addr, available) =>
+ (0 until available).map(_ => addr)
+ }.toSeq
/**
* Sequence of currently assigned resource addresses.
Review comment:
can you add comment saying can see the same addr multiple times when
slotsPeraddress > 1
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
