This is an automated email from the ASF dual-hosted git repository. vanzin pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new dd0725d [SPARK-28679][YARN] changes to setResourceInformation to handle empty resources and reflection error handling dd0725d is described below commit dd0725d7eaf2f0dc9fb3b13326b07c32812b0ad9 Author: Alessandro Bellina <abell...@nvidia.com> AuthorDate: Mon Aug 26 12:00:33 2019 -0700 [SPARK-28679][YARN] changes to setResourceInformation to handle empty resources and reflection error handling ## What changes were proposed in this pull request? This fixes issues that can arise when the jars for different hadoop versions mix, and short-circuits the case where we are running with a spark that was not built for yarn 3 (resource support). ## How was this patch tested? I tested it manually. Closes #25403 from abellina/SPARK-28679. Authored-by: Alessandro Bellina <abell...@nvidia.com> Signed-off-by: Marcelo Vanzin <van...@cloudera.com> --- .../spark/deploy/yarn/ResourceRequestHelper.scala | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala index 522c16b..f524962 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/ResourceRequestHelper.scala @@ -143,17 +143,28 @@ private object ResourceRequestHelper extends Logging { require(resource != null, "Resource parameter should not be null!") logDebug(s"Custom resources requested: $resources") + if (resources.isEmpty) { + // no point in going forward, as we don't have anything to set + return + } + if (!isYarnResourceTypesAvailable()) { - if (resources.nonEmpty) { - logWarning("Ignoring custom resource requests because " + - "the version of YARN does not support it!") - } + logWarning("Ignoring custom resource requests because " + + "the version of YARN does not support it!") return } val resInfoClass = Utils.classForName(RESOURCE_INFO_CLASS) val setResourceInformationMethod = - resource.getClass.getMethod("setResourceInformation", classOf[String], resInfoClass) + try { + resource.getClass.getMethod("setResourceInformation", classOf[String], resInfoClass) + } catch { + case e: NoSuchMethodException => + throw new SparkException( + s"Cannot find setResourceInformation in ${resource.getClass}. " + + "This is likely due to a JAR conflict between different YARN versions.", e) + } + resources.foreach { case (name, rawAmount) => try { val AMOUNT_AND_UNIT_REGEX(amountPart, unitPart) = rawAmount --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org