Repository: spark Updated Branches: refs/heads/master 0ee9fd9e5 -> 40eeef952
[SPARK-15046][YARN] Parse value of token renewal interval correctly. Use the config variable definition both to set and parse the value, avoiding issues with code expecting the value in a different format. Tested by running spark-submit with --principal / --keytab. Author: Marcelo Vanzin <[email protected]> Closes #13669 from vanzin/SPARK-15046. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/40eeef95 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/40eeef95 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/40eeef95 Branch: refs/heads/master Commit: 40eeef95256b0740d759d921f0385023f0b91666 Parents: 0ee9fd9 Author: Marcelo Vanzin <[email protected]> Authored: Wed Jun 15 09:09:21 2016 -0500 Committer: Tom Graves <[email protected]> Committed: Wed Jun 15 09:09:21 2016 -0500 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala | 4 ++-- .../main/scala/org/apache/spark/internal/config/package.scala | 7 +++++++ yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala | 5 ----- 3 files changed, 9 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/40eeef95/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala index 7a5fc86..bb1793d 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala @@ -41,6 +41,7 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdenti import org.apache.spark.{SparkConf, SparkException} import org.apache.spark.annotation.DeveloperApi import org.apache.spark.internal.Logging +import org.apache.spark.internal.config._ import org.apache.spark.util.Utils /** @@ -288,8 +289,7 @@ class SparkHadoopUtil extends Logging { credentials: Credentials): Long = { val now = System.currentTimeMillis() - val renewalInterval = - sparkConf.getLong("spark.yarn.token.renewal.interval", (24 hours).toMillis) + val renewalInterval = sparkConf.get(TOKEN_RENEWAL_INTERVAL).get credentials.getAllTokens.asScala .filter(_.getKind == DelegationTokenIdentifier.HDFS_DELEGATION_KIND) http://git-wip-us.apache.org/repos/asf/spark/blob/40eeef95/core/src/main/scala/org/apache/spark/internal/config/package.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 2c1e0b7..05dd683 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -17,6 +17,8 @@ package org.apache.spark.internal +import java.util.concurrent.TimeUnit + import org.apache.spark.launcher.SparkLauncher import org.apache.spark.network.util.ByteUnit @@ -80,6 +82,11 @@ package object config { .doc("Name of the Kerberos principal.") .stringConf.createOptional + private[spark] val TOKEN_RENEWAL_INTERVAL = ConfigBuilder("spark.yarn.token.renewal.interval") + .internal() + .timeConf(TimeUnit.MILLISECONDS) + .createOptional + private[spark] val EXECUTOR_INSTANCES = ConfigBuilder("spark.executor.instances") .intConf .createOptional http://git-wip-us.apache.org/repos/asf/spark/blob/40eeef95/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala ---------------------------------------------------------------------- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala index c4dd320..ad2412e 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala @@ -243,11 +243,6 @@ package object config { .toSequence .createWithDefault(Nil) - private[spark] val TOKEN_RENEWAL_INTERVAL = ConfigBuilder("spark.yarn.token.renewal.interval") - .internal() - .timeConf(TimeUnit.MILLISECONDS) - .createOptional - /* Private configs. */ private[spark] val CREDENTIALS_FILE_PATH = ConfigBuilder("spark.yarn.credentials.file") --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
