tgravescs commented on a change in pull request #26682: [SPARK-29306][CORE]
Stage Level Sched: Executors need to track what ResourceProfile they are
created with
URL: https://github.com/apache/spark/pull/26682#discussion_r365979892
##########
File path:
core/src/test/scala/org/apache/spark/resource/ResourceProfileSuite.scala
##########
@@ -18,72 +18,98 @@
package org.apache.spark.resource
import org.apache.spark.{SparkConf, SparkFunSuite}
-import org.apache.spark.internal.config._
+import org.apache.spark.internal.config.{EXECUTOR_CORES, EXECUTOR_MEMORY,
EXECUTOR_MEMORY_OVERHEAD, SPARK_EXECUTOR_PREFIX}
+import org.apache.spark.internal.config.Python.PYSPARK_EXECUTOR_MEMORY
class ResourceProfileSuite extends SparkFunSuite {
override def afterEach() {
try {
- ResourceProfile.resetDefaultProfile(new SparkConf)
+ ResourceProfile.clearDefaultProfile
} finally {
super.afterEach()
}
}
-
test("Default ResourceProfile") {
val rprof = ResourceProfile.getOrCreateDefaultProfile(new SparkConf)
assert(rprof.id === ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
assert(rprof.executorResources.size === 2,
"Executor resources should contain cores and memory by default")
assert(rprof.executorResources(ResourceProfile.CORES).amount === 1,
s"Executor resources should have 1 core")
+ assert(rprof.getExecutorCores.get === 1,
+ s"Executor resources should have 1 core")
assert(rprof.executorResources(ResourceProfile.MEMORY).amount === 1024,
s"Executor resources should have 1024 memory")
+ assert(rprof.executorResources.get(ResourceProfile.PYSPARK_MEM) == None,
+ s"pyspark memory empty if not specified")
+ assert(rprof.executorResources.get(ResourceProfile.OVERHEAD_MEM) == None,
+ s"overhead memory empty if not specified")
assert(rprof.taskResources.size === 1,
"Task resources should just contain cpus by default")
assert(rprof.taskResources(ResourceProfile.CPUS).amount === 1,
s"Task resources should have 1 cpu")
+ assert(rprof.getTaskCpus.get === 1,
+ s"Task resources should have 1 cpu")
Review comment:
thanks, I'll clean them up in the entire file
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
