Github user galv commented on a diff in the pull request:
https://github.com/apache/spark/pull/20761#discussion_r189473608
--- Diff:
resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/ClientSuite.scala
---
@@ -199,6 +204,88 @@ class ClientSuite extends SparkFunSuite with Matchers {
appContext.getMaxAppAttempts should be (42)
}
+ test("Resource type args propagate, resource type not defined") {
+ assume(ResourceTypeHelper.isYarnResourceTypesAvailable())
+ val sparkConf = new SparkConf()
+ .set(YARN_AM_RESOURCE_TYPES_PREFIX + "gpu", "121m")
+ val args = new ClientArguments(Array())
+
+ val appContext =
Records.newRecord(classOf[ApplicationSubmissionContext])
+ val getNewApplicationResponse =
Records.newRecord(classOf[GetNewApplicationResponse])
+ val containerLaunchContext =
Records.newRecord(classOf[ContainerLaunchContext])
+
+ val client = new Client(args, sparkConf)
+
+ try {
+ client.createApplicationSubmissionContext(
+ new YarnClientApplication(getNewApplicationResponse, appContext),
+ containerLaunchContext)
+ } catch {
+ case NonFatal(e) =>
+ val expectedExceptionClass =
"org.apache.hadoop.yarn.exceptions.ResourceNotFoundException"
+ if (e.getClass.getName != expectedExceptionClass) {
+ fail(s"Exception caught: $e is not an instance of
$expectedExceptionClass!")
+ }
+ }
+ }
+
+ test("Resource type args propagate (client mode)") {
+ assume(ResourceTypeHelper.isYarnResourceTypesAvailable())
+ TestYarnResourceTypeHelper.initializeResourceTypes(List("gpu", "fpga"))
+
+ val sparkConf = new SparkConf()
+ .set(YARN_AM_RESOURCE_TYPES_PREFIX + "gpu", "121m")
--- End diff --
Nitpick: It makes no sense to request "121m" of a GPU. GPUs can be acquired
only as discrete devices. I understand no actual acquisition is happening right
now, but these unit tests might mislead newcomers. I would use a different
resource name, one that intentionally shows it is nonsensical like
"some_resource_with_units1"
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
