kiszk commented on a change in pull request #26682: [SPARK-29306][CORE] Stage
Level Sched: Executors need to track what ResourceProfile they are created with
URL: https://github.com/apache/spark/pull/26682#discussion_r365977319
##########
File path:
core/src/test/scala/org/apache/spark/resource/ResourceProfileSuite.scala
##########
@@ -18,72 +18,98 @@
package org.apache.spark.resource
import org.apache.spark.{SparkConf, SparkFunSuite}
-import org.apache.spark.internal.config._
+import org.apache.spark.internal.config.{EXECUTOR_CORES, EXECUTOR_MEMORY,
EXECUTOR_MEMORY_OVERHEAD, SPARK_EXECUTOR_PREFIX}
+import org.apache.spark.internal.config.Python.PYSPARK_EXECUTOR_MEMORY
class ResourceProfileSuite extends SparkFunSuite {
override def afterEach() {
try {
- ResourceProfile.resetDefaultProfile(new SparkConf)
+ ResourceProfile.clearDefaultProfile
} finally {
super.afterEach()
}
}
-
test("Default ResourceProfile") {
val rprof = ResourceProfile.getOrCreateDefaultProfile(new SparkConf)
assert(rprof.id === ResourceProfile.DEFAULT_RESOURCE_PROFILE_ID)
assert(rprof.executorResources.size === 2,
"Executor resources should contain cores and memory by default")
assert(rprof.executorResources(ResourceProfile.CORES).amount === 1,
s"Executor resources should have 1 core")
+ assert(rprof.getExecutorCores.get === 1,
+ s"Executor resources should have 1 core")
assert(rprof.executorResources(ResourceProfile.MEMORY).amount === 1024,
s"Executor resources should have 1024 memory")
+ assert(rprof.executorResources.get(ResourceProfile.PYSPARK_MEM) == None,
+ s"pyspark memory empty if not specified")
+ assert(rprof.executorResources.get(ResourceProfile.OVERHEAD_MEM) == None,
+ s"overhead memory empty if not specified")
assert(rprof.taskResources.size === 1,
"Task resources should just contain cpus by default")
assert(rprof.taskResources(ResourceProfile.CPUS).amount === 1,
s"Task resources should have 1 cpu")
+ assert(rprof.getTaskCpus.get === 1,
+ s"Task resources should have 1 cpu")
Review comment:
nit: we may drop `s` from line 39 to 53.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
