Repository: spark Updated Branches: refs/heads/branch-2.2 f8c83fdc5 -> bf8163f5b
[SPARK-22319][CORE][BACKPORT-2.2] call loginUserFromKeytab before accessing hdfs In SparkSubmit, call loginUserFromKeytab before attempting to make RPC calls to the NameNode. Same as #https://github.com/apache/spark/pull/19540, but for branch-2.2. Manually tested for master as described in https://github.com/apache/spark/pull/19540. Author: Steven Rand <sr...@palantir.com> Closes #19554 from sjrand/SPARK-22319-branch-2.2. Change-Id: Ic550a818fd6a3f38b356ac48029942d463738458 Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bf8163f5 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bf8163f5 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bf8163f5 Branch: refs/heads/branch-2.2 Commit: bf8163f5be55a94e02849ccbaf755702a2c6c68f Parents: f8c83fd Author: Steven Rand <sr...@palantir.com> Authored: Mon Oct 23 14:26:03 2017 +0800 Committer: jerryshao <ss...@hortonworks.com> Committed: Mon Oct 23 14:26:03 2017 +0800 ---------------------------------------------------------------------- .../org/apache/spark/deploy/SparkSubmit.scala | 38 ++++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/bf8163f5/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index 86d578e..4f2f2c1 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -316,6 +316,25 @@ object SparkSubmit extends CommandLineUtils { RPackageUtils.checkAndBuildRPackage(args.jars, printStream, args.verbose) } + // assure a keytab is available from any place in a JVM + if (clusterManager == YARN || clusterManager == LOCAL) { + if (args.principal != null) { + require(args.keytab != null, "Keytab must be specified when principal is specified") + if (!new File(args.keytab).exists()) { + throw new SparkException(s"Keytab file: ${args.keytab} does not exist") + } else { + // Add keytab and principal configurations in sysProps to make them available + // for later use; e.g. in spark sql, the isolated class loader used to talk + // to HiveMetastore will use these settings. They will be set as Java system + // properties and then loaded by SparkConf + sysProps.put("spark.yarn.keytab", args.keytab) + sysProps.put("spark.yarn.principal", args.principal) + + UserGroupInformation.loginUserFromKeytab(args.principal, args.keytab) + } + } + } + // In client mode, download remote files. var localPrimaryResource: String = null var localJars: String = null @@ -582,25 +601,6 @@ object SparkSubmit extends CommandLineUtils { } } - // assure a keytab is available from any place in a JVM - if (clusterManager == YARN || clusterManager == LOCAL) { - if (args.principal != null) { - require(args.keytab != null, "Keytab must be specified when principal is specified") - if (!new File(args.keytab).exists()) { - throw new SparkException(s"Keytab file: ${args.keytab} does not exist") - } else { - // Add keytab and principal configurations in sysProps to make them available - // for later use; e.g. in spark sql, the isolated class loader used to talk - // to HiveMetastore will use these settings. They will be set as Java system - // properties and then loaded by SparkConf - sysProps.put("spark.yarn.keytab", args.keytab) - sysProps.put("spark.yarn.principal", args.principal) - - UserGroupInformation.loginUserFromKeytab(args.principal, args.keytab) - } - } - } - // In yarn-cluster mode, use yarn.Client as a wrapper around the user class if (isYarnCluster) { childMainClass = "org.apache.spark.deploy.yarn.Client" --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org