CDH-53414: Sentry fails to setup secure connection to HMS if the service unix user is missing active tgt
Looking at the code of Sentry and HiveMetaStoreClient, it looks like this problem has been there for a long time. But it's mostly masked by the restarts we are doing after the clean deployment. Some context on how our internal deployment works: CDEP as part of deployment kicks off the first run of all services via CM and once it is done, it kinit's service principals under the service unix users for e.g: [email protected] gets activated under hdfs user. So during the course of CM first run, sentry and other services don't have any keytabs active under local unix user and during this time, the log of sentry is cluttered with the above errors. But after we do a restart after CDEP completes, the Sentry code picks up the principal that was activated under "sentry" user and thus works properly. Ideally the service processes shouldn't depend on the keytabs active on running unix user but rather use the keytabs supplied by CM in the process directory. Looking at the code of HMSFollower we invoke the classes of Hive - HiveMetaStoreClient which exclusively uses the UserGroupInformation object. And for UserGroupInformation object to pick up the keytab, one should explicitly call the methods like loginUserFromKeytabAndReturnUGI or use getUGIFromSubject. For the fact that HMSFollower already logs into the keytab, we can leverage the method getUGIFromSubject() to make the UserGroupInformation aware of keytab authentication. If UGI object is not made aware of keytab, then the invocation of UserGroupInformation#getCurrentUser falls back to Unix user. Adding extra logic to HMSFollower to make UserGroupInformation aware of the keytab that way it uses it for communication with HMS. Change-Id: I520ef1183b55e286386d401ce6c1b3946a648d18 Reviewed-on: http://gerrit.sjc.cloudera.com:8080/22294 Reviewed-by: Alexander Kolbasov <[email protected]> Tested-by: Alexander Kolbasov <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/ee488318 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/ee488318 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/ee488318 Branch: refs/for/cdh5-1.5.1_ha Commit: ee488318228b454e0398ab9225ef9bbd7a63624c Parents: 3f9e25e Author: Vamsee Yarlagadda <[email protected]> Authored: Wed May 3 21:55:50 2017 -0700 Committer: Alexander Kolbasov <[email protected]> Committed: Thu May 4 00:06:17 2017 -0700 ---------------------------------------------------------------------- .../org/apache/sentry/service/thrift/HMSFollower.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/ee488318/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java ---------------------------------------------------------------------- diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java index 122da84..c3f821c 100644 --- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java +++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SaslRpcServer; import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.hcatalog.messaging.HCatEventMessage; import org.apache.sentry.binding.hive.conf.HiveAuthzConf; import org.apache.sentry.core.common.exception.SentryInvalidHMSEventException; @@ -46,12 +47,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.sentry.binding.metastore.messaging.json.*; -import javax.security.auth.Subject; import javax.security.auth.login.LoginException; import java.io.File; import java.io.IOException; import java.net.SocketException; -import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.util.List; import java.util.Map; @@ -118,7 +117,7 @@ public class HMSFollower implements Runnable { * Throws @MetaException if there was a problem on creating an HMSClient */ private HiveMetaStoreClient getMetaStoreClient(Configuration conf) - throws LoginException, MetaException, PrivilegedActionException { + throws IOException, InterruptedException, LoginException, MetaException { if(client != null) { return client; } @@ -159,9 +158,12 @@ public class HMSFollower implements Runnable { // Instantiating SentryKerberosContext in non-server mode handles the ticket renewal. kerberosContext = new SentryKerberosContext(principal, keytab, false); + UserGroupInformation.setConfiguration(hiveConf); + UserGroupInformation clientUGI = UserGroupInformation.getUGIFromSubject(kerberosContext.getSubject()); + // HiveMetaStoreClient handles the connection retry logic to HMS and can be configured using properties: // hive.metastore.connect.retries, hive.metastore.client.connect.retry.delay - client = Subject.doAs(kerberosContext.getSubject(), new PrivilegedExceptionAction<HiveMetaStoreClient>() { + client = clientUGI.doAs(new PrivilegedExceptionAction<HiveMetaStoreClient>() { @Override public HiveMetaStoreClient run() throws Exception { return new HiveMetaStoreClient(hiveConf); @@ -172,9 +174,6 @@ public class HMSFollower implements Runnable { // Kerberos login failed LOGGER.error("Failed to setup kerberos context."); throw e; - } catch (PrivilegedActionException e) { - LOGGER.error("Failed to setup secure connection to HMS."); - throw e; } finally { // Shutdown kerberos context if HMS connection failed to setup to avoid thread leaks. if ((kerberosContext != null) && (client == null)) {
