This is an automated email from the ASF dual-hosted git repository. yangjie01 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new e32a8249c6dd [SPARK-46918][YARN] Replace self-defined variables with Hadoop ContainerExitStatus e32a8249c6dd is described below commit e32a8249c6ddb15e01d2307964f2978f4a10ad56 Author: Cheng Pan <cheng...@apache.org> AuthorDate: Tue Jan 30 20:17:11 2024 +0800 [SPARK-46918][YARN] Replace self-defined variables with Hadoop ContainerExitStatus ### What changes were proposed in this pull request? Replace the Spark self-defined `VMEM_EXCEEDED_EXIT_CODE` and `PMEM_EXCEEDED_EXIT_CODE` with Hadoop defined `ContainerExitStatus.KILLED_EXCEEDED_VMEM` and `ContainerExitStatus.KILLED_EXCEEDED_PMEM` which were introduced in YARN-2091(since Hadoop 2.5.0) ### Why are the changes needed? Minor code clean-up ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass GA. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #44950 from pan3793/SPARK-46918. Authored-by: Cheng Pan <cheng...@apache.org> Signed-off-by: yangjie01 <yangji...@baidu.com> --- .../main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index 736eaa52b81c..7f0469937fef 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -851,9 +851,6 @@ private[yarn] class YarnAllocator( onHostStr, completedContainer.getState, completedContainer.getExitStatus)) - // Hadoop 2.2.X added a ContainerExitStatus we should switch to use - // there are some exit status' we shouldn't necessarily count against us, but for - // now I think its ok as none of the containers are expected to exit. val exitStatus = completedContainer.getExitStatus val (exitCausedByApp, containerExitReason) = exitStatus match { case _ if shutdown => @@ -867,7 +864,7 @@ private[yarn] class YarnAllocator( // just as easily finish on any other executor. See SPARK-8167. (false, s"Container ${containerId}${onHostStr} was preempted.") // Should probably still count memory exceeded exit codes towards task failures - case VMEM_EXCEEDED_EXIT_CODE => + case ContainerExitStatus.KILLED_EXCEEDED_VMEM => val vmemExceededPattern = raw"$MEM_REGEX of $MEM_REGEX virtual memory used".r val diag = vmemExceededPattern.findFirstIn(completedContainer.getDiagnostics) .map(_.concat(".")).getOrElse("") @@ -876,7 +873,7 @@ private[yarn] class YarnAllocator( s"${YarnConfiguration.NM_VMEM_PMEM_RATIO} or disabling " + s"${YarnConfiguration.NM_VMEM_CHECK_ENABLED} because of YARN-4714." (true, message) - case PMEM_EXCEEDED_EXIT_CODE => + case ContainerExitStatus.KILLED_EXCEEDED_PMEM => val pmemExceededPattern = raw"$MEM_REGEX of $MEM_REGEX physical memory used".r val diag = pmemExceededPattern.findFirstIn(completedContainer.getDiagnostics) .map(_.concat(".")).getOrElse("") @@ -1025,8 +1022,6 @@ private[yarn] class YarnAllocator( private object YarnAllocator { val MEM_REGEX = "[0-9.]+ [KMG]B" - val VMEM_EXCEEDED_EXIT_CODE = -103 - val PMEM_EXCEEDED_EXIT_CODE = -104 val DECOMMISSIONING_NODES_CACHE_SIZE = 200 val NOT_APP_AND_SYSTEM_FAULT_EXIT_STATUS = Set( --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org