tgravescs commented on a change in pull request #30204:
URL: https://github.com/apache/spark/pull/30204#discussion_r523078747
##########
File path: core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
##########
@@ -293,20 +305,32 @@ object ResourceProfile extends Logging {
private def getDefaultExecutorResources(conf: SparkConf): Map[String,
ExecutorResourceRequest] = {
val ereqs = new ExecutorResourceRequests()
- ereqs.cores(conf.get(EXECUTOR_CORES))
- ereqs.memory(conf.get(EXECUTOR_MEMORY).toString)
- conf.get(EXECUTOR_MEMORY_OVERHEAD).map(mem =>
ereqs.memoryOverhead(mem.toString))
- conf.get(PYSPARK_EXECUTOR_MEMORY).map(mem =>
ereqs.pysparkMemory(mem.toString))
- if (conf.get(MEMORY_OFFHEAP_ENABLED)) {
- // Explicitly add suffix b as default unit of offHeapMemory is Mib
- ereqs.offHeapMemory(conf.get(MEMORY_OFFHEAP_SIZE).toString + "b")
- }
+ val cores = conf.get(EXECUTOR_CORES)
+ ereqs.cores(cores)
+ val memory = conf.get(EXECUTOR_MEMORY)
+ ereqs.memory(memory.toString)
+ val overheadMem = conf.get(EXECUTOR_MEMORY_OVERHEAD)
+ overheadMem.map(mem => ereqs.memoryOverhead(mem.toString))
+ val pysparkMem = conf.get(PYSPARK_EXECUTOR_MEMORY)
+ pysparkMem.map(mem => ereqs.pysparkMemory(mem.toString))
+ val offheapMem = Utils.executorOffHeapMemorySizeAsMb(conf)
+ ereqs.offHeapMemory(offheapMem.toString)
val execReq = ResourceUtils.parseAllResourceRequests(conf,
SPARK_EXECUTOR_PREFIX)
+ val customResources = new mutable.HashMap[String, ExecutorResourceRequest]
execReq.foreach { req =>
val name = req.id.resourceName
+ customResources(name) =
+ new ExecutorResourceRequest(
+ req.id.resourceName,
+ req.amount,
+ req.discoveryScript.orElse(""),
+ req.vendor.orElse(""))
ereqs.resource(name, req.amount, req.discoveryScript.orElse(""),
req.vendor.orElse(""))
}
Review comment:
we could but its less efficient you have to iterate over execReq a
second time and then over all the requests we just added, now that should be
very fast, but just thought since already iterating over it just do it at same
time. If you prefer it this way I can change it?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
