Repository: spark Updated Branches: refs/heads/master 2fcb9cb95 -> 5fd53c64b
[SPARK-9833] [YARN] Add options to disable delegation token retrieval. This allows skipping the code that tries to talk to Hive and HBase to fetch delegation tokens, in case that somehow conflicts with the application being run. Author: Marcelo Vanzin <[email protected]> Closes #8134 from vanzin/SPARK-9833. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5fd53c64 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5fd53c64 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5fd53c64 Branch: refs/heads/master Commit: 5fd53c64bb01de74ae57a7068de85b34adc856cf Parents: 2fcb9cb Author: Marcelo Vanzin <[email protected]> Authored: Wed Aug 19 10:51:59 2015 -0700 Committer: Marcelo Vanzin <[email protected]> Committed: Wed Aug 19 10:51:59 2015 -0700 ---------------------------------------------------------------------- docs/running-on-yarn.md | 12 +++++++++ .../org/apache/spark/deploy/yarn/Client.scala | 27 +++++++++++++++----- 2 files changed, 33 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/5fd53c64/docs/running-on-yarn.md ---------------------------------------------------------------------- diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 8ac26e9..5159ef9 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -369,6 +369,18 @@ If you need a reference to the proper location to put log files in the YARN so t See <code>spark.yarn.config.gatewayPath</code>. </td> </tr> +<tr> + <td><code>spark.yarn.security.tokens.${service}.enabled</code></td> + <td>true</td> + <td> + Controls whether to retrieve delegation tokens for non-HDFS services when security is enabled. + By default, delegation tokens for all supported services are retrieved when those services are + configured, but it's possible to disable that behavior if it somehow conflicts with the + application being run. + <p/> + Currently supported services are: hive, hbase + </td> +</tr> </table> # Important notes http://git-wip-us.apache.org/repos/asf/spark/blob/5fd53c64/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala ---------------------------------------------------------------------- diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 262c6a8..bff585b 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -285,8 +285,8 @@ private[spark] class Client( // multiple times, YARN will fail to launch containers for the app with an internal // error. val distributedUris = new HashSet[String] - obtainTokenForHiveMetastore(hadoopConf, credentials) - obtainTokenForHBase(hadoopConf, credentials) + obtainTokenForHiveMetastore(sparkConf, hadoopConf, credentials) + obtainTokenForHBase(sparkConf, hadoopConf, credentials) val replication = sparkConf.getInt("spark.yarn.submit.file.replication", fs.getDefaultReplication(dst)).toShort @@ -1239,8 +1239,11 @@ object Client extends Logging { /** * Obtains token for the Hive metastore and adds them to the credentials. */ - private def obtainTokenForHiveMetastore(conf: Configuration, credentials: Credentials) { - if (UserGroupInformation.isSecurityEnabled) { + private def obtainTokenForHiveMetastore( + sparkConf: SparkConf, + conf: Configuration, + credentials: Credentials) { + if (shouldGetTokens(sparkConf, "hive") && UserGroupInformation.isSecurityEnabled) { val mirror = universe.runtimeMirror(getClass.getClassLoader) try { @@ -1297,8 +1300,11 @@ object Client extends Logging { /** * Obtain security token for HBase. */ - def obtainTokenForHBase(conf: Configuration, credentials: Credentials): Unit = { - if (UserGroupInformation.isSecurityEnabled) { + def obtainTokenForHBase( + sparkConf: SparkConf, + conf: Configuration, + credentials: Credentials): Unit = { + if (shouldGetTokens(sparkConf, "hbase") && UserGroupInformation.isSecurityEnabled) { val mirror = universe.runtimeMirror(getClass.getClassLoader) try { @@ -1394,4 +1400,13 @@ object Client extends Logging { components.mkString(Path.SEPARATOR) } + /** + * Return whether delegation tokens should be retrieved for the given service when security is + * enabled. By default, tokens are retrieved, but that behavior can be changed by setting + * a service-specific configuration. + */ + def shouldGetTokens(conf: SparkConf, service: String): Boolean = { + conf.getBoolean(s"spark.yarn.security.tokens.${service}.enabled", true) + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
