kiszk commented on a change in pull request #26682: [SPARK-29306][CORE] Stage
Level Sched: Executors need to track what ResourceProfile they are created with
URL: https://github.com/apache/spark/pull/26682#discussion_r365977509
##########
File path:
core/src/test/scala/org/apache/spark/resource/ResourceProfileSuite.scala
##########
@@ -18,72 +18,98 @@
package org.apache.spark.resource
import org.apache.spark.{SparkConf, SparkFunSuite}
-import org.apache.spark.internal.config._
+import org.apache.spark.internal.config.{EXECUTOR_CORES, EXECUTOR_MEMORY,
EXECUTOR_MEMORY_OVERHEAD, SPARK_EXECUTOR_PREFIX}
+import org.apache.spark.internal.config.Python.PYSPARK_EXECUTOR_MEMORY
class ResourceProfileSuite extends SparkFunSuite {
override def afterEach() {
try {
- ResourceProfile.resetDefaultProfile(new SparkConf)
+ ResourceProfile.clearDefaultProfile
} finally {
super.afterEach()
}
}
-
test("Default ResourceProfile") {
val rprof = ResourceProfile.getOrCreateDefaultProfile(new SparkConf)
assert(rprof.id === ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
assert(rprof.executorResources.size === 2,
"Executor resources should contain cores and memory by default")
assert(rprof.executorResources(ResourceProfile.CORES).amount === 1,
s"Executor resources should have 1 core")
+ assert(rprof.getExecutorCores.get === 1,
+ s"Executor resources should have 1 core")
assert(rprof.executorResources(ResourceProfile.MEMORY).amount === 1024,
s"Executor resources should have 1024 memory")
+ assert(rprof.executorResources.get(ResourceProfile.PYSPARK_MEM) == None,
+ s"pyspark memory empty if not specified")
+ assert(rprof.executorResources.get(ResourceProfile.OVERHEAD_MEM) == None,
+ s"overhead memory empty if not specified")
assert(rprof.taskResources.size === 1,
"Task resources should just contain cpus by default")
assert(rprof.taskResources(ResourceProfile.CPUS).amount === 1,
s"Task resources should have 1 cpu")
+ assert(rprof.getTaskCpus.get === 1,
+ s"Task resources should have 1 cpu")
}
test("Default ResourceProfile with app level resources specified") {
val conf = new SparkConf
+ conf.set(PYSPARK_EXECUTOR_MEMORY.key, "2g")
+ conf.set(EXECUTOR_MEMORY_OVERHEAD.key, "1g")
+ conf.set(EXECUTOR_MEMORY.key, "4g")
+ conf.set(EXECUTOR_CORES.key, "4")
conf.set("spark.task.resource.gpu.amount", "1")
conf.set(s"$SPARK_EXECUTOR_PREFIX.resource.gpu.amount", "1")
conf.set(s"$SPARK_EXECUTOR_PREFIX.resource.gpu.discoveryScript",
"nameOfScript")
val rprof = ResourceProfile.getOrCreateDefaultProfile(conf)
assert(rprof.id === ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
val execResources = rprof.executorResources
- assert(execResources.size === 3,
+ assert(execResources.size === 5,
"Executor resources should contain cores, memory, and gpu " +
execResources)
+ assert(execResources.contains("gpu"), "Executor resources should have gpu")
+ assert(rprof.executorResources(ResourceProfile.CORES).amount === 4,
+ s"Executor resources should have 4 core")
+ assert(rprof.getExecutorCores.get === 4,
+ s"Executor resources should have 4 core")
+ assert(rprof.executorResources(ResourceProfile.MEMORY).amount === 4096,
+ s"Executor resources should have 1024 memory")
+ assert(rprof.executorResources(ResourceProfile.PYSPARK_MEM).amount == 2048,
+ s"pyspark memory empty if not specified")
+ assert(rprof.executorResources(ResourceProfile.OVERHEAD_MEM).amount ==
1024,
+ s"overhead memory empty if not specified")
Review comment:
ditto
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
