Ngone51 commented on code in PR #36716:
URL: https://github.com/apache/spark/pull/36716#discussion_r891178775
##########
core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala:
##########
@@ -530,6 +535,87 @@ class MasterSuite extends SparkFunSuite
schedulingWithEverything(spreadOut = false)
}
+ test("scheduling for app with multiple resource profiles") {
+ scheduleExecutorsForAppWithMultiRPs(withMaxCores = false)
+ }
+
+ test("scheduling for app with multiple resource profiles with max cores") {
+ scheduleExecutorsForAppWithMultiRPs(withMaxCores = true)
+ }
+
+ private def scheduleExecutorsForAppWithMultiRPs(withMaxCores: Boolean): Unit
= {
+ val appInfo: ApplicationInfo = if (withMaxCores) {
+ makeAppInfo(
+ 1024, maxCores = Some(30), initialExecutorLimit = Some(0))
+ } else {
+ makeAppInfo(
+ 1024, maxCores = None, initialExecutorLimit = Some(0))
+ }
+
+ val master = makeAliveMaster()
+ val conf = new SparkConf()
+ val workers = (1 to 4).map { idx =>
+ val worker = new MockWorker(master.self, conf)
+ worker.rpcEnv.setupEndpoint(s"worker-$idx", worker)
+ val workerReg = RegisterWorker(
+ worker.id,
+ "localhost",
+ worker.self.address.port,
+ worker.self,
+ 10,
+ 4096,
+ "http://localhost:8080";,
+ RpcAddress("localhost", 10000))
+ master.self.send(workerReg)
+ worker
+ }
+
+ // Register app and schedule.
+ master.registerApplication(appInfo)
+ startExecutorsOnWorkers(master)
+ assert(appInfo.executors.isEmpty)
+
+ // Request executors with multiple resource profile.
+ val rp1 = DeployTestUtils.createResourceProfile(Some(2048), Map.empty,
Some(5))
+ val rp2 = DeployTestUtils.createResourceProfile(Some(2048), Map.empty,
Some(10))
Review Comment:
Could you also test the case where no worker can satisfy the resource
profile ? In this case, no executor for that rp should be launched.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
