Repository: hadoop
Updated Branches:
refs/heads/trunk ff013d2c9 -> 7dd26d537
YARN-8308. Fixed YARN Service AM failure with HDFS token renewal.
Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7dd26d53
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7dd26d53
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7dd26d53
Branch: refs/heads/trunk
Commit: 7dd26d5378483cdbce1dc594e8650a96c2976281
Parents: ff013d2
Author: Eric Yang <[email protected]>
Authored: Thu May 31 21:06:25 2018 -0400
Committer: Eric Yang <[email protected]>
Committed: Thu May 31 21:09:36 2018 -0400
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceMaster.java | 56 ++++++++++++++------
.../yarn/service/client/ServiceClient.java | 11 ++++
2 files changed, 50 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd26d53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index 28881ac..0caa119 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -71,8 +71,14 @@ public class ServiceMaster extends CompositeService {
LoggerFactory.getLogger(ServiceMaster.class);
public static final String YARNFILE_OPTION = "yarnfile";
-
- private static String serviceDefPath;
+ public static final String SERVICE_NAME_OPTION = "service_name";
+ public static final String KEYTAB_OPTION = "keytab";
+ public static final String PRINCIPAL_NAME_OPTION = "principal_name";
+
+ private String serviceDefPath;
+ private String serviceName;
+ private String serviceKeytab;
+ private String servicePrincipalName;
protected ServiceContext context;
public ServiceMaster(String name) {
@@ -85,15 +91,24 @@ public class ServiceMaster extends CompositeService {
context = new ServiceContext();
Path appDir = getAppDir();
context.serviceHdfsDir = appDir.toString();
+ context.tokens = recordTokensForContainers();
+ Credentials credentials = null;
+ if (UserGroupInformation.isSecurityEnabled()) {
+ credentials = UserGroupInformation.getCurrentUser().getCredentials();
+ doSecureLogin();
+ }
SliderFileSystem fs = new SliderFileSystem(conf);
- context.fs = fs;
fs.setAppDir(appDir);
+ context.fs = fs;
loadApplicationJson(context, fs);
-
- context.tokens = recordTokensForContainers();
if (UserGroupInformation.isSecurityEnabled()) {
- doSecureLogin();
+ // add back the credentials
+ if (credentials != null) {
+ UserGroupInformation.getCurrentUser().addCredentials(credentials);
+ }
+ removeHdfsDelegationToken(UserGroupInformation.getLoginUser());
}
+
// Take yarn config from YarnFile and merge them into YarnConfiguration
for (Map.Entry<String, String> entry : context.service
.getConfiguration().getProperties().entrySet()) {
@@ -157,13 +172,12 @@ public class ServiceMaster extends CompositeService {
private void doSecureLogin()
throws IOException, URISyntaxException {
// read the localized keytab specified by user
- File keytab = new File(String.format(KEYTAB_LOCATION,
- context.service.getName()));
+ File keytab = new File(String.format(KEYTAB_LOCATION, getServiceName()));
if (!keytab.exists()) {
LOG.info("No keytab localized at " + keytab);
// Check if there exists a pre-installed keytab at host
- String preInstalledKeytab = context.service.getKerberosPrincipal()
- .getKeytab();
+ String preInstalledKeytab = context.service == null ? this.serviceKeytab
+ : context.service.getKerberosPrincipal().getKeytab();
if (!StringUtils.isEmpty(preInstalledKeytab)) {
URI uri = new URI(preInstalledKeytab);
if (uri.getScheme().equals("file")) {
@@ -177,29 +191,24 @@ public class ServiceMaster extends CompositeService {
LOG.info("No keytab exists: " + keytab);
return;
}
- String principal = context.service.getKerberosPrincipal()
- .getPrincipalName();
+ String principal = context.service == null ? this.servicePrincipalName
+ : context.service.getKerberosPrincipal().getPrincipalName();
if (StringUtils.isEmpty((principal))) {
principal = UserGroupInformation.getLoginUser().getShortUserName();
LOG.info("No principal name specified. Will use AM " +
"login identity {} to attempt keytab-based login", principal);
}
- Credentials credentials = UserGroupInformation.getCurrentUser()
- .getCredentials();
LOG.info("User before logged in is: " + UserGroupInformation
.getCurrentUser());
String principalName = SecurityUtil.getServerPrincipal(principal,
ServiceUtils.getLocalHostName(getConfig()));
UserGroupInformation.loginUserFromKeytab(principalName,
keytab.getAbsolutePath());
- // add back the credentials
- UserGroupInformation.getCurrentUser().addCredentials(credentials);
LOG.info("User after logged in is: " + UserGroupInformation
.getCurrentUser());
context.principal = principalName;
context.keytab = keytab.getAbsolutePath();
- removeHdfsDelegationToken(UserGroupInformation.getLoginUser());
}
// Remove HDFS delegation token from login user and ensure AM to use keytab
@@ -231,6 +240,10 @@ public class ServiceMaster extends CompositeService {
return new Path(serviceDefPath).getParent();
}
+ protected String getServiceName() {
+ return serviceName;
+ }
+
protected ServiceScheduler createServiceScheduler(ServiceContext context)
throws IOException, YarnException {
return new ServiceScheduler(context);
@@ -310,9 +323,18 @@ public class ServiceMaster extends CompositeService {
opts.addOption(YARNFILE_OPTION, true, "HDFS path to JSON service " +
"specification");
opts.getOption(YARNFILE_OPTION).setRequired(true);
+ opts.addOption(SERVICE_NAME_OPTION, true, "Service name");
+ opts.getOption(SERVICE_NAME_OPTION).setRequired(true);
+ opts.addOption(KEYTAB_OPTION, true, "Service AM keytab");
+ opts.addOption(PRINCIPAL_NAME_OPTION, true,
+ "Service AM keytab principal");
GenericOptionsParser parser = new GenericOptionsParser(conf, opts, args);
CommandLine cmdLine = parser.getCommandLine();
serviceMaster.serviceDefPath = cmdLine.getOptionValue(YARNFILE_OPTION);
+ serviceMaster.serviceName = cmdLine.getOptionValue(SERVICE_NAME_OPTION);
+ serviceMaster.serviceKeytab = cmdLine.getOptionValue(KEYTAB_OPTION);
+ serviceMaster.servicePrincipalName = cmdLine
+ .getOptionValue(PRINCIPAL_NAME_OPTION);
serviceMaster.init(conf);
serviceMaster.start();
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd26d53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index e86ecbc..c86f5de 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -871,6 +871,17 @@ public class ServiceClient extends AppAdminClient
implements SliderExitCodes,
//TODO debugAM CLI.add(Arguments.ARG_DEBUG)
CLI.add("-" + ServiceMaster.YARNFILE_OPTION, new Path(appRootDir,
app.getName() + ".json"));
+ CLI.add("-" + ServiceMaster.SERVICE_NAME_OPTION, app.getName());
+ if (app.getKerberosPrincipal() != null) {
+ if (!StringUtils.isEmpty(app.getKerberosPrincipal().getKeytab())) {
+ CLI.add("-" + ServiceMaster.KEYTAB_OPTION,
+ app.getKerberosPrincipal().getKeytab());
+ }
+ if (!StringUtils.isEmpty(app.getKerberosPrincipal().getPrincipalName()))
{
+ CLI.add("-" + ServiceMaster.PRINCIPAL_NAME_OPTION,
+ app.getKerberosPrincipal().getPrincipalName());
+ }
+ }
// pass the registry binding
CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
