Repository: hive Updated Branches: refs/heads/master 4d98fc7b5 -> 623ecaa53
HIVE-17368 : DBTokenStore fails to connect in Kerberos enabled remote HMS environment (Vihang Karajgaonkar, reviewed by Aihua Xu and Janaki Lahorani) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/623ecaa5 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/623ecaa5 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/623ecaa5 Branch: refs/heads/master Commit: 623ecaa53bc7d559a4181383ecea25ce0868cbdc Parents: 4d98fc7 Author: Vihang Karajgaonkar <vih...@cloudera.com> Authored: Fri Sep 1 19:14:35 2017 -0700 Committer: Vihang Karajgaonkar <vih...@cloudera.com> Committed: Sun Oct 22 17:29:01 2017 -0700 ---------------------------------------------------------------------- .../org/apache/hive/minikdc/MiniHiveKdc.java | 34 ++++++++++++++++ .../hive/minikdc/TestJdbcWithDBTokenStore.java | 6 ++- .../security/TestHadoopAuthBridge23.java | 3 +- .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 42 +++++++++++++++----- .../hadoop/hive/ql/session/SessionState.java | 16 +++++--- .../cli/session/HiveSessionImplwithUGI.java | 2 + .../hive/metastore/security/DBTokenStore.java | 4 +- .../security/DelegationTokenSecretManager.java | 13 +++--- .../MetastoreDelegationTokenManager.java | 15 +++---- 9 files changed, 103 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/623ecaa5/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java ---------------------------------------------------------------------- diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java index bbec37e..c02879d 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java @@ -51,6 +51,7 @@ public class MiniHiveKdc { public static String HIVE_TEST_USER_2 = "user2"; public static String HIVE_TEST_SUPER_USER = "superuser"; public static String AUTHENTICATION_TYPE = "KERBEROS"; + private static final String HIVE_METASTORE_SERVICE_PRINCIPAL = "hive"; private final MiniKdc miniKdc; private final File workDir; @@ -204,6 +205,39 @@ public class MiniHiveKdc { return getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf, AUTHENTICATION_TYPE); } + public static MiniHS2 getMiniHS2WithKerbWithRemoteHMSWithKerb(MiniHiveKdc miniHiveKdc, + HiveConf hiveConf) throws Exception { + return getMiniHS2WithKerbWithRemoteHMSWithKerb(miniHiveKdc, hiveConf, AUTHENTICATION_TYPE); + } + + /** + * Create a MiniHS2 with the hive service principal and keytab in MiniHiveKdc. It uses remote HMS + * and can support a different Sasl authType. It creates a metastore service principal and keytab + * which can be used for secure HMS + * @param miniHiveKdc + * @param hiveConf + * @param authenticationType + * @return new MiniHS2 instance + * @throws Exception + */ + private static MiniHS2 getMiniHS2WithKerbWithRemoteHMSWithKerb(MiniHiveKdc miniHiveKdc, + HiveConf hiveConf, String authenticationType) throws Exception { + String hivePrincipal = + miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL); + String hiveKeytab = miniHiveKdc.getKeyTabFile( + miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL)); + + String hiveMetastorePrincipal = + miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_METASTORE_SERVICE_PRINCIPAL); + String hiveMetastoreKeytab = miniHiveKdc.getKeyTabFile( + miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_METASTORE_SERVICE_PRINCIPAL)); + + return new MiniHS2.Builder().withConf(hiveConf) + .withSecureRemoteMetastore(hiveMetastorePrincipal, hiveMetastoreKeytab). + withMiniKdc(hivePrincipal, hiveKeytab).withAuthenticationType(authenticationType) + .build(); + } + /** * Create a MiniHS2 with the hive service principal and keytab in MiniHiveKdc. It uses remote HMS * and can support a different Sasl authType http://git-wip-us.apache.org/repos/asf/hive/blob/623ecaa5/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java index 8695232..ab25c3b 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java @@ -40,7 +40,11 @@ public class TestJdbcWithDBTokenStore extends TestJdbcWithMiniKdc{ //using old config value tests backwards compatibility hiveConf.setVar(ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS, "org.apache.hadoop.hive.thrift.DBTokenStore"); miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf); - miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf); + miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMSWithKerb(miniHiveKdc, hiveConf); miniHS2.start(confOverlay); + String metastorePrincipal = miniHS2.getConfProperty(ConfVars.METASTORE_KERBEROS_PRINCIPAL.varname); + String hs2Principal = miniHS2.getConfProperty(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL.varname); + String hs2KeyTab = miniHS2.getConfProperty(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB.varname); + System.out.println("HS2 principal : " + hs2Principal + " HS2 keytab : " + hs2KeyTab + " Metastore principal : " + metastorePrincipal); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/623ecaa5/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java index cedc933..8094f92 100644 --- a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java +++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java @@ -168,7 +168,8 @@ public class TestHadoopAuthBridge23 { tokenManager.startThreads(); tokenManager.stopThreads(); - String tokenStrForm = tokenManager.getDelegationToken(clientUgi.getShortUserName()); + String tokenStrForm = + tokenManager.getDelegationToken(clientUgi.getShortUserName(), clientUgi.getShortUserName()); Token<DelegationTokenIdentifier> t= new Token<DelegationTokenIdentifier>(); t.decodeFromUrlString(tokenStrForm); http://git-wip-us.apache.org/repos/asf/hive/blob/623ecaa5/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 5702de2..e554cc8 100644 --- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -73,6 +73,7 @@ public class MiniHS2 extends AbstractHiveService { private final String serverPrincipal; private final boolean isMetastoreRemote; private final boolean cleanupLocalDirOnStartup; + private final boolean isMetastoreSecure; private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY; public enum MiniClusterType { @@ -94,6 +95,9 @@ public class MiniHS2 extends AbstractHiveService { private String authType = "KERBEROS"; private boolean isHA = false; private boolean cleanupLocalDirOnStartup = true; + private boolean isMetastoreSecure; + private String metastoreServerPrincipal; + private String metastoreServerKeyTab; public Builder() { } @@ -120,6 +124,14 @@ public class MiniHS2 extends AbstractHiveService { return this; } + public Builder withSecureRemoteMetastore(String metastoreServerPrincipal, String metastoreServerKeyTab) { + this.isMetastoreRemote = true; + this.isMetastoreSecure = true; + this.metastoreServerPrincipal = metastoreServerPrincipal; + this.metastoreServerKeyTab = metastoreServerKeyTab; + return this; + } + public Builder withConf(HiveConf hiveConf) { this.hiveConf = hiveConf; return this; @@ -154,7 +166,8 @@ public class MiniHS2 extends AbstractHiveService { hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE); } return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab, - isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup); + isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup, + isMetastoreSecure, metastoreServerPrincipal, metastoreServerKeyTab); } } @@ -192,7 +205,10 @@ public class MiniHS2 extends AbstractHiveService { private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useMiniKdc, String serverPrincipal, String serverKeytab, boolean isMetastoreRemote, - boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup) throws Exception { + boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup, + boolean isMetastoreSecure, + String metastoreServerPrincipal, + String metastoreKeyTab) throws Exception { // Always use localhost for hostname as some tests like SSL CN validation ones // are tied to localhost being present in the certificate name super( @@ -209,6 +225,7 @@ public class MiniHS2 extends AbstractHiveService { this.useMiniKdc = useMiniKdc; this.serverPrincipal = serverPrincipal; this.isMetastoreRemote = isMetastoreRemote; + this.isMetastoreSecure = isMetastoreSecure; this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup; baseDir = getBaseDir(); localFS = FileSystem.getLocal(hiveConf); @@ -262,9 +279,15 @@ public class MiniHS2 extends AbstractHiveService { hiveConf.setVar(ConfVars.HIVE_SERVER2_KERBEROS_KEYTAB, serverKeytab); hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, authType); } - String metaStoreURL = - "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator - + "test_metastore;create=true"; + + String metaStoreURL = "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator + + "test_metastore;create=true"; + + if (isMetastoreSecure) { + hiveConf.setVar(ConfVars.METASTORE_KERBEROS_PRINCIPAL, metastoreServerPrincipal); + hiveConf.setVar(ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, metastoreKeyTab); + hiveConf.setBoolVar(ConfVars.METASTORE_USE_THRIFT_SASL, true); + } fs.mkdirs(baseFsDir); Path wareHouseDir = new Path(baseFsDir, "warehouse"); @@ -302,10 +325,11 @@ public class MiniHS2 extends AbstractHiveService { this(hiveConf, clusterType, false); } - public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, - boolean usePortsFromConf) throws Exception { - this(hiveConf, clusterType, false, null, null, false, usePortsFromConf, - "KERBEROS", false, true); + public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsFromConf) + throws Exception { + this(hiveConf, clusterType, false, null, null, + false, usePortsFromConf, "KERBEROS", false, true, + false, null, null); } public void start(Map<String, String> confOverlay) throws Exception { http://git-wip-us.apache.org/repos/asf/hive/blob/623ecaa5/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 2bdb719..bb6ddc6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.log.ProgressMonitor; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.cache.CachedStore; @@ -1754,12 +1755,15 @@ public class SessionState { private void unCacheDataNucleusClassLoaders() { try { - Hive threadLocalHive = Hive.get(sessionConf); - if ((threadLocalHive != null) && (threadLocalHive.getMSC() != null) - && (threadLocalHive.getMSC().isLocalMetaStore())) { - if (sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL).equals(ObjectStore.class.getName()) - || sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL).equals(CachedStore.class.getName()) && - sessionConf.getVar(ConfVars.METASTORE_CACHED_RAW_STORE_IMPL).equals(ObjectStore.class.getName())) { + boolean isLocalMetastore = + HiveConfUtil.isEmbeddedMetaStore(sessionConf.getVar(HiveConf.ConfVars.METASTOREURIS)); + if (isLocalMetastore) { + if (sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL) + .equals(ObjectStore.class.getName()) || + sessionConf.getVar(ConfVars.METASTORE_RAW_STORE_IMPL) + .equals(CachedStore.class.getName()) && sessionConf + .getVar(ConfVars.METASTORE_CACHED_RAW_STORE_IMPL) + .equals(ObjectStore.class.getName())) { ObjectStore.unCacheDataNucleusClassLoaders(); } } http://git-wip-us.apache.org/repos/asf/hive/blob/623ecaa5/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java index 8000a5b..8975aee 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java @@ -124,6 +124,8 @@ public class HiveSessionImplwithUGI extends HiveSessionImpl { if (hmsDelegationTokenStr != null) { try { Hive.get(getHiveConf()).cancelDelegationToken(hmsDelegationTokenStr); + hmsDelegationTokenStr = null; + getHiveConf().setVar(HiveConf.ConfVars.METASTORE_TOKEN_SIGNATURE, ""); } catch (HiveException e) { throw new HiveSQLException("Couldn't cancel delegation token", e); } http://git-wip-us.apache.org/repos/asf/hive/blob/623ecaa5/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DBTokenStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DBTokenStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DBTokenStore.java index 2f26010..ad6d751 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DBTokenStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DBTokenStore.java @@ -152,9 +152,7 @@ public class DBTokenStore implements DelegationTokenStore { tokenStore = handler.getClass().getMethod("getMS").invoke(handler); break; case HIVESERVER2: - Object hiveObject = ((Class<?>) handler) - .getMethod("get", org.apache.hadoop.conf.Configuration.class, java.lang.Class.class) - .invoke(handler, conf, DBTokenStore.class); + Object hiveObject = ((Class<?>) handler).getMethod("get").invoke(handler, null); tokenStore = ((Class<?>) handler).getMethod("getMSC").invoke(hiveObject); break; default: http://git-wip-us.apache.org/repos/asf/hive/blob/623ecaa5/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java index aae96a5..a719f06 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/DelegationTokenSecretManager.java @@ -97,12 +97,15 @@ public class DelegationTokenSecretManager return renewToken(t, user); } - public synchronized String getDelegationToken(String renewer) throws IOException { - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - Text owner = new Text(ugi.getUserName()); + public synchronized String getDelegationToken(final String ownerStr, final String renewer) throws IOException { + if (ownerStr == null) { + throw new RuntimeException("Delegation token owner is null"); + } + Text owner = new Text(ownerStr); Text realUser = null; - if (ugi.getRealUser() != null) { - realUser = new Text(ugi.getRealUser().getUserName()); + UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser(); + if (currentUgi.getUserName() != null) { + realUser = new Text(currentUgi.getUserName()); } DelegationTokenIdentifier ident = new DelegationTokenIdentifier(owner, new Text(renewer), realUser); http://git-wip-us.apache.org/repos/asf/hive/blob/623ecaa5/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java index 3cfdd8a..8d84fd7 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java @@ -99,13 +99,14 @@ public class MetastoreDelegationTokenManager { ownerUgi = UserGroupInformation.createProxyUser(owner, UserGroupInformation.getCurrentUser()); ProxyUsers.authorize(ownerUgi, remoteAddr, null); } - return ownerUgi.doAs(new PrivilegedExceptionAction<String>() { - - @Override - public String run() throws IOException { - return secretManager.getDelegationToken(renewer); - } - }); + //if impersonation is turned on this called using the HiveSessionImplWithUGI + //using sessionProxy. so the currentUser will be the impersonated user here eg. oozie + //we cannot create a proxy user which represents Oozie's client user here since + //we cannot authenticate it using Kerberos/Digest. We trust the user which opened + //session using Kerberos in this case. + //if impersonation is turned off, the current user is Hive which can open + //kerberos connections to HMS if required. + return secretManager.getDelegationToken(owner, renewer); } public String getDelegationTokenWithService(String owner, String renewer, String service, String remoteAddr)
