Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/14065#discussion_r69988867
--- Diff:
yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnSparkHadoopUtil.scala ---
@@ -96,237 +87,19 @@ class YarnSparkHadoopUtil extends SparkHadoopUtil {
if (credentials != null) credentials.getSecretKey(new Text(key)) else
null
}
- /**
- * Get the list of namenodes the user may access.
- */
- def getNameNodesToAccess(sparkConf: SparkConf): Set[Path] = {
- sparkConf.get(NAMENODES_TO_ACCESS)
- .map(new Path(_))
- .toSet
- }
-
- def getTokenRenewer(conf: Configuration): String = {
- val delegTokenRenewer = Master.getMasterPrincipal(conf)
- logDebug("delegation token renewer is: " + delegTokenRenewer)
- if (delegTokenRenewer == null || delegTokenRenewer.length() == 0) {
- val errorMessage = "Can't get Master Kerberos principal for use as
renewer"
- logError(errorMessage)
- throw new SparkException(errorMessage)
- }
- delegTokenRenewer
- }
-
- /**
- * Obtains tokens for the namenodes passed in and adds them to the
credentials.
- */
- def obtainTokensForNamenodes(
- paths: Set[Path],
- conf: Configuration,
- creds: Credentials,
- renewer: Option[String] = None
- ): Unit = {
- if (UserGroupInformation.isSecurityEnabled()) {
- val delegTokenRenewer = renewer.getOrElse(getTokenRenewer(conf))
- paths.foreach { dst =>
- val dstFs = dst.getFileSystem(conf)
- logInfo("getting token for namenode: " + dst)
- dstFs.addDelegationTokens(delegTokenRenewer, creds)
- }
- }
- }
-
- /**
- * Obtains token for the Hive metastore and adds them to the credentials.
- */
- def obtainTokenForHiveMetastore(
- sparkConf: SparkConf,
- conf: Configuration,
- credentials: Credentials) {
- if (shouldGetTokens(sparkConf, "hive") &&
UserGroupInformation.isSecurityEnabled) {
- YarnSparkHadoopUtil.get.obtainTokenForHiveMetastore(conf).foreach {
- credentials.addToken(new Text("hive.server2.delegation.token"), _)
- }
- }
- }
-
- /**
- * Obtain a security token for HBase.
- */
- def obtainTokenForHBase(
- sparkConf: SparkConf,
- conf: Configuration,
- credentials: Credentials): Unit = {
- if (shouldGetTokens(sparkConf, "hbase") &&
UserGroupInformation.isSecurityEnabled) {
- YarnSparkHadoopUtil.get.obtainTokenForHBase(conf).foreach { token =>
- credentials.addToken(token.getService, token)
- logInfo("Added HBase security token to credentials.")
- }
- }
- }
-
- /**
- * Return whether delegation tokens should be retrieved for the given
service when security is
- * enabled. By default, tokens are retrieved, but that behavior can be
changed by setting
- * a service-specific configuration.
- */
- private def shouldGetTokens(conf: SparkConf, service: String): Boolean =
{
- conf.getBoolean(s"spark.yarn.security.tokens.${service}.enabled", true)
- }
-
private[spark] override def
startExecutorDelegationTokenRenewer(sparkConf: SparkConf): Unit = {
- tokenRenewer = Some(new ExecutorDelegationTokenUpdater(sparkConf,
conf))
- tokenRenewer.get.updateCredentialsIfRequired()
+ configurableTokenManager(sparkConf).delegationTokenUpdater(conf)
--- End diff --
I find this syntax a little confusing. You're calling
`configurableTokenManager(sparkConf)` in a bunch of different places. To me
that looks like either:
- each call is creating a new token manager
- there's some cache of token managers somewhere keyed by the spark
configuration passed here
Neither sounds good to me. And the actual implementation is actually
neither: there's a single token manager singleton that is instantiated in the
first call to `configurableTokenManager`.
Why doesn't `Client` instantiate a token manager in its constructor
instead? Another option is to have an explicit method in
`ConfigurableTokenManager` to initialize the singleton, although I'm not a fan
of singletons in general.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
