HIVE-12270: Add DBTokenStore support to HS2 delegation token (Chaoyu Tang, reviewed by Szehon Ho)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/87131d0c Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/87131d0c Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/87131d0c Branch: refs/heads/master Commit: 87131d0c7cce973d8792ed354f70ec73b0f52dcd Parents: 41e8201 Author: ctang <[email protected]> Authored: Tue Mar 8 15:05:54 2016 -0500 Committer: ctang <[email protected]> Committed: Tue Mar 8 15:05:54 2016 -0500 ---------------------------------------------------------------------- .../org/apache/hive/minikdc/MiniHiveKdc.java | 15 + .../hive/minikdc/TestJdbcWithDBTokenStore.java | 40 + .../hive/minikdc/TestJdbcWithMiniKdc.java | 12 +- .../hadoop/hive/thrift/TestDBTokenStore.java | 3 +- metastore/if/hive_metastore.thrift | 24 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 22340 ++++++++++------- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 1409 +- .../ThriftHiveMetastore_server.skeleton.cpp | 40 + .../hive/metastore/api/ThriftHiveMetastore.java | 9220 ++++++- .../gen-php/metastore/ThriftHiveMetastore.php | 1756 ++ .../hive_metastore/ThriftHiveMetastore-remote | 56 + .../hive_metastore/ThriftHiveMetastore.py | 1639 +- .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 450 + .../hadoop/hive/metastore/HiveMetaStore.java | 159 + .../hive/metastore/HiveMetaStoreClient.java | 42 + .../hadoop/hive/metastore/IMetaStoreClient.java | 19 +- .../gen-py/TCLIService/TCLIService-remote | 0 .../hive/service/auth/HiveAuthFactory.java | 33 +- .../apache/hadoop/hive/thrift/DBTokenStore.java | 49 +- .../hive/thrift/HiveDelegationTokenManager.java | 1 + 20 files changed, 26383 insertions(+), 10924 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/87131d0c/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java ---------------------------------------------------------------------- diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java index dedbf35..4e3a9c5 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java @@ -179,5 +179,20 @@ public class MiniHiveKdc { withMiniKdc(hivePrincipal, hiveKeytab).build(); } + /** + * Create a MiniHS2 with the hive service principal and keytab in MiniHiveKdc + * @param miniHiveKdc + * @param hiveConf + * @return new MiniHS2 instance + * @throws Exception + */ + public static MiniHS2 getMiniHS2WithKerbWithRemoteHMS(MiniHiveKdc miniHiveKdc, HiveConf hiveConf) throws Exception { + String hivePrincipal = + miniHiveKdc.getFullyQualifiedServicePrincipal(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL); + String hiveKeytab = miniHiveKdc.getKeyTabFile( + miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_SERVICE_PRINCIPAL)); + return new MiniHS2.Builder().withConf(hiveConf).withRemoteMetastore(). + withMiniKdc(hivePrincipal, hiveKeytab).build(); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/87131d0c/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java new file mode 100644 index 0000000..d690aaa --- /dev/null +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithDBTokenStore.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.minikdc; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.BeforeClass; + +public class TestJdbcWithDBTokenStore extends TestJdbcWithMiniKdc{ + + @BeforeClass + public static void beforeTest() throws Exception { + Class.forName(MiniHS2.getJdbcDriverName()); + confOverlay.put(ConfVars.HIVE_SERVER2_SESSION_HOOK.varname, + SessionHookTest.class.getName()); + + HiveConf hiveConf = new HiveConf(); + hiveConf.setVar(ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS, "org.apache.hadoop.hive.thrift.DBTokenStore"); + miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf); + miniHS2 = MiniHiveKdc.getMiniHS2WithKerbWithRemoteHMS(miniHiveKdc, hiveConf); + miniHS2.start(confOverlay); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/87131d0c/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java ---------------------------------------------------------------------- diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java index 3ef2ce3..71a08fb 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java @@ -57,10 +57,10 @@ public class TestJdbcWithMiniKdc { } } - private static MiniHS2 miniHS2 = null; - private static MiniHiveKdc miniHiveKdc = null; - private static Map<String, String> confOverlay = new HashMap<String, String>(); - private Connection hs2Conn; + protected static MiniHS2 miniHS2 = null; + protected static MiniHiveKdc miniHiveKdc = null; + protected static Map<String, String> confOverlay = new HashMap<String, String>(); + protected Connection hs2Conn; @BeforeClass public static void beforeTest() throws Exception { @@ -241,7 +241,7 @@ public class TestJdbcWithMiniKdc { * @param expectedValue * @throws Exception */ - private void verifyProperty(String propertyName, String expectedValue) throws Exception { + protected void verifyProperty(String propertyName, String expectedValue) throws Exception { Statement stmt = hs2Conn .createStatement(); ResultSet res = stmt.executeQuery("set " + propertyName); assertTrue(res.next()); @@ -251,7 +251,7 @@ public class TestJdbcWithMiniKdc { } // Store the given token in the UGI - private void storeToken(String tokenStr, UserGroupInformation ugi) + protected void storeToken(String tokenStr, UserGroupInformation ugi) throws Exception { Utils.setTokenStr(ugi, tokenStr, HiveAuthFactory.HS2_CLIENT_TOKEN); http://git-wip-us.apache.org/repos/asf/hive/blob/87131d0c/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java index f5934ee..4bfa224 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.thrift.DelegationTokenStore.TokenStoreException; +import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge.Server.ServerMode; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation; import org.apache.hadoop.security.token.delegation.HiveDelegationTokenSupport; @@ -37,7 +38,7 @@ public class TestDBTokenStore extends TestCase{ public void testDBTokenStore() throws TokenStoreException, MetaException, IOException { DelegationTokenStore ts = new DBTokenStore(); - ts.init(new HMSHandler("Test handler"), null); + ts.init(new HMSHandler("Test handler"), ServerMode.METASTORE); assertEquals(0, ts.getMasterKeys().length); assertEquals(false,ts.removeMasterKey(-1)); try{ http://git-wip-us.apache.org/repos/asf/hive/blob/87131d0c/metastore/if/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift index 3635054..6a55962 100755 --- a/metastore/if/hive_metastore.thrift +++ b/metastore/if/hive_metastore.thrift @@ -1294,6 +1294,30 @@ service ThriftHiveMetastore extends fb303.FacebookService // method to cancel delegation token obtained from metastore server void cancel_delegation_token(1:string token_str_form) throws (1:MetaException o1) + // add a delegation token + bool add_token(1:string token_identifier, 2:string delegation_token) + + // remove a delegation token + bool remove_token(1:string token_identifier) + + // get a delegation token by identifier + string get_token(1:string token_identifier) + + // get all delegation token identifiers + list<string> get_all_token_identifiers() + + // add master key + i32 add_master_key(1:string key) throws (1:MetaException o1) + + // update master key + void update_master_key(1:i32 seq_number, 2:string key) throws (1:NoSuchObjectException o1, 2:MetaException o2) + + // remove master key + bool remove_master_key(1:i32 key_seq) + + // get master keys + list<string> get_master_keys() + // Transaction and lock management calls // Get just list of open transactions GetOpenTxnsResponse get_open_txns()
