This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 87ccfc24023 [SPARK-43196][YARN] Replace reflection w/ direct calling
for `ContainerLaunchContext#setTokensConf`
87ccfc24023 is described below
commit 87ccfc2402394d168a90c8df494482c7a1d6a552
Author: yangjie01 <yangji...@baidu.com>
AuthorDate: Wed Apr 19 21:09:18 2023 -0700
[SPARK-43196][YARN] Replace reflection w/ direct calling for
`ContainerLaunchContext#setTokensConf`
### What changes were proposed in this pull request?
This pr replace reflection with direct calling for
`ContainerLaunchContext#setTokensConf` in
`o.a.s.deploy.yarn.Client#setTokenConf` function.
### Why are the changes needed?
SPARK-37205 uses reflection to call `ContainerLaunchContext#setTokensConf`
for compatibility with Hadoop 2.7, since SPARK-42452 removed support for
Hadoop2, we can call it directly instead of using reflection.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass GA.
Closes #40855 from LuciferYang/SPARK-43196.
Authored-by: yangjie01 <yangji...@baidu.com>
Signed-off-by: Chao Sun <sunc...@apple.com>
---
.../main/scala/org/apache/spark/deploy/yarn/Client.scala | 15 +--------------
1 file changed, 1 insertion(+), 14 deletions(-)
diff --git
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 06d8b4e2509..7010067e1ae 100644
---
a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++
b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -77,10 +77,6 @@ private[spark] class Client(
private val isClusterMode = sparkConf.get(SUBMIT_DEPLOY_MODE) == "cluster"
- // ContainerLaunchContext.setTokensConf is only available in Hadoop 2.9+ and
3.x, so here we use
- // reflection to avoid compilation for Hadoop 2.7 profile.
- private val SET_TOKENS_CONF_METHOD = "setTokensConf"
-
private val isClientUnmanagedAMEnabled = sparkConf.get(YARN_UNMANAGED_AM) &&
!isClusterMode
private var appMaster: ApplicationMaster = _
private var stagingDirPath: Path = _
@@ -382,16 +378,7 @@ private[spark] class Client(
}
copy.write(dob);
- // since this method was added in Hadoop 2.9 and 3.0, we use reflection
here to avoid
- // compilation error for Hadoop 2.7 profile.
- val setTokensConfMethod = try {
- amContainer.getClass.getMethod(SET_TOKENS_CONF_METHOD,
classOf[ByteBuffer])
- } catch {
- case _: NoSuchMethodException =>
- throw new SparkException(s"Cannot find setTokensConf method in
${amContainer.getClass}." +
- s" Please check YARN version and make sure it is 2.9+ or 3.x")
- }
- setTokensConfMethod.invoke(amContainer, ByteBuffer.wrap(dob.getData))
+ amContainer.setTokensConf(ByteBuffer.wrap(dob.getData))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
