Repository: hive Updated Branches: refs/heads/master 06fb30235 -> a3f0f99d5
HIVE-20388 : Move common classes out of metastore-server (Alexander Kolbasov reviewed by Vihang Karajgaonkar) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3f0f99d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3f0f99d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3f0f99d Branch: refs/heads/master Commit: a3f0f99d53ab916143c49cb8611f701bfa70e59c Parents: 06fb302 Author: Alexander Kolbasov <[email protected]> Authored: Mon Aug 27 14:41:12 2018 -0700 Committer: Vihang Karajgaonkar <[email protected]> Committed: Mon Aug 27 14:41:12 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/lockmgr/DbTxnManager.java | 7 +- .../hive/ql/stats/StatsUpdaterThread.java | 3 +- .../ql/util/HiveStrictManagedMigration.java | 3 +- .../hive/service/auth/HiveAuthFactory.java | 10 +- .../hive/metastore/txn/TxnCommonUtils.java | 127 +++++++++++++++++++ .../hadoop/hive/metastore/HiveAlterHandler.java | 8 +- .../hadoop/hive/metastore/HiveMetaStore.java | 4 +- .../hive/metastore/HiveMetaStoreClient.java | 8 +- .../MaterializationsRebuildLockCleanerTask.java | 3 +- .../MetastoreDelegationTokenManager.java | 6 +- .../hadoop/hive/metastore/txn/TxnHandler.java | 2 +- .../hadoop/hive/metastore/txn/TxnUtils.java | 104 +-------------- .../metastore/utils/MetaStoreServerUtils.java | 85 ++++++++++++- .../hive/metastore/utils/MetaStoreUtils.java | 46 +------ .../hive/metastore/utils/SecurityUtils.java | 53 +------- .../HiveMetaStoreClientPreCatalog.java | 8 +- 16 files changed, 246 insertions(+), 231 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java index f746d2d..4e3d74c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java @@ -18,13 +18,11 @@ package org.apache.hadoop.hive.ql.lockmgr; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidTxnWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.LockRequestBuilder; @@ -37,7 +35,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchTxnException; import org.apache.hadoop.hive.metastore.api.TxnAbortedException; import org.apache.hadoop.hive.metastore.api.TxnToWriteId; import org.apache.hadoop.hive.metastore.api.CommitTxnRequest; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryPlan; @@ -50,7 +48,6 @@ import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc; import org.apache.hadoop.hive.ql.plan.LockTableDesc; import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc; import org.apache.hadoop.hive.ql.plan.UnlockTableDesc; -import org.apache.hadoop.hive.ql.session.*; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.common.util.ShutdownHookManager; import org.apache.thrift.TException; @@ -676,7 +673,7 @@ public final class DbTxnManager extends HiveTxnManagerImpl { assert isTxnOpen(); assert validTxnList != null && !validTxnList.isEmpty(); try { - return TxnUtils.createValidTxnWriteIdList( + return TxnCommonUtils.createValidTxnWriteIdList( txnId, getMS().getValidWriteIds(tableList, validTxnList)); } catch (TException e) { throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(), e); http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java index a50ec18..981f6ae 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUpdaterThread.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.StatsUpdateMode; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.ql.DriverUtils; @@ -500,7 +501,7 @@ public class StatsUpdaterThread extends Thread implements MetaStoreThread { // TODO: acid utils don't support catalogs GetValidWriteIdsRequest req = new GetValidWriteIdsRequest( Lists.newArrayList(fullTableName.getDbTable()), null); - return TxnUtils.createValidReaderWriteIdList( + return TxnCommonUtils.createValidReaderWriteIdList( txnHandler.getValidWriteIds(req).getTblValidWriteIds().get(0)); } http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java index 54df541..13a8af7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.TransactionalValidationListener; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.HiveStrictManagedUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -994,7 +995,7 @@ public class HiveStrictManagedMigration { String fqn = table.getDbName() + "." + table.getTableName(); List<TableValidWriteIds> writeIdsObj = msc.getValidWriteIds( Lists.newArrayList(fqn), txns.toString()); - String validWriteIds = TxnUtils.createValidTxnWriteIdList(txnId, writeIdsObj) + String validWriteIds = TxnCommonUtils.createValidTxnWriteIdList(txnId, writeIdsObj) .getTableValidWriteIdList(fqn).writeToString(); long writeId = msc.allocateTableWriteId(txnId, table.getDbName(), table.getTableName()); result = new TxnCtx(writeId, validWriteIds, txnId); http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java ---------------------------------------------------------------------- diff --git a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java index d23e438..f5f6d49 100644 --- a/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java +++ b/service/src/java/org/apache/hive/service/auth/HiveAuthFactory.java @@ -27,21 +27,15 @@ import javax.security.auth.login.LoginException; import javax.security.sasl.AuthenticationException; import javax.security.sasl.Sasl; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.conf.HiveConfUtil; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.metastore.security.DBTokenStore; -import org.apache.hadoop.hive.metastore.security.DelegationTokenStore; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; -import org.apache.hadoop.hive.metastore.security.MemoryTokenStore; import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; -import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore; -import org.apache.hadoop.hive.metastore.utils.SecurityUtils; import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -97,7 +91,7 @@ public class HiveAuthFactory { delegationTokenManager = new MetastoreDelegationTokenManager(); try { Object baseHandler = null; - String tokenStoreClass = SecurityUtils.getTokenStoreClassName(conf); + String tokenStoreClass = MetaStoreServerUtils.getTokenStoreClassName(conf); if (tokenStoreClass.equals(DBTokenStore.class.getName())) { // IMetaStoreClient is needed to access token store if DBTokenStore is to be used. It http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java new file mode 100644 index 0000000..94cb18d --- /dev/null +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.txn; + +import org.apache.hadoop.hive.common.ValidReadTxnList; +import org.apache.hadoop.hive.common.ValidReaderWriteIdList; +import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.common.ValidTxnWriteIdList; +import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse; +import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; + +import java.util.BitSet; +import java.util.Collections; +import java.util.List; + +public class TxnCommonUtils { + /** + * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a + * {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to + * read the files, and thus treats both open and aborted transactions as invalid. + * @param txns txn list from the metastore + * @param currentTxn Current transaction that the user has open. If this is greater than 0 it + * will be removed from the exceptions list so that the user sees his own + * transaction as valid. + * @return a valid txn list. + */ + public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) { + /* + * The highWaterMark should be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0 + * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which + * doesn't make sense for Snapshot Isolation. Of course for Read Committed, the list should + * include the latest committed set. + */ + long highWaterMark = (currentTxn > 0) ? Math.min(currentTxn, txns.getTxn_high_water_mark()) + : txns.getTxn_high_water_mark(); + + // Open txns are already sorted in ascending order. This list may or may not include HWM + // but it is guaranteed that list won't have txn > HWM. But, if we overwrite the HWM with currentTxn + // then need to truncate the exceptions list accordingly. + List<Long> openTxns = txns.getOpen_txns(); + + // We care only about open/aborted txns below currentTxn and hence the size should be determined + // for the exceptions list. The currentTxn will be missing in openTxns list only in rare case like + // txn is aborted by AcidHouseKeeperService and compactor actually cleans up the aborted txns. + // So, for such cases, we get negative value for sizeToHwm with found position for currentTxn, and so, + // we just negate it to get the size. + int sizeToHwm = (currentTxn > 0) ? Collections.binarySearch(openTxns, currentTxn) : openTxns.size(); + sizeToHwm = (sizeToHwm < 0) ? (-sizeToHwm) : sizeToHwm; + long[] exceptions = new long[sizeToHwm]; + BitSet inAbortedBits = BitSet.valueOf(txns.getAbortedBits()); + BitSet outAbortedBits = new BitSet(); + long minOpenTxnId = Long.MAX_VALUE; + int i = 0; + for (long txn : openTxns) { + // For snapshot isolation, we don't care about txns greater than current txn and so stop here. + // Also, we need not include current txn to exceptions list. + if ((currentTxn > 0) && (txn >= currentTxn)) { + break; + } + if (inAbortedBits.get(i)) { + outAbortedBits.set(i); + } else if (minOpenTxnId == Long.MAX_VALUE) { + minOpenTxnId = txn; + } + exceptions[i++] = txn; + } + return new ValidReadTxnList(exceptions, outAbortedBits, highWaterMark, minOpenTxnId); + } + + /** + * Transform a {@link org.apache.hadoop.hive.metastore.api.GetValidWriteIdsResponse} to a + * {@link org.apache.hadoop.hive.common.ValidTxnWriteIdList}. This assumes that the caller intends to + * read the files, and thus treats both open and aborted transactions as invalid. + * @param currentTxnId current txn ID for which we get the valid write ids list + * @param list valid write ids list from the metastore + * @return a valid write IDs list for the whole transaction. + */ + public static ValidTxnWriteIdList createValidTxnWriteIdList(Long currentTxnId, + List<TableValidWriteIds> validIds) { + ValidTxnWriteIdList validTxnWriteIdList = new ValidTxnWriteIdList(currentTxnId); + for (TableValidWriteIds tableWriteIds : validIds) { + validTxnWriteIdList.addTableValidWriteIdList(createValidReaderWriteIdList(tableWriteIds)); + } + return validTxnWriteIdList; + } + + /** + * Transform a {@link TableValidWriteIds} to a + * {@link org.apache.hadoop.hive.common.ValidReaderWriteIdList}. This assumes that the caller intends to + * read the files, and thus treats both open and aborted write ids as invalid. + * @param tableWriteIds valid write ids for the given table from the metastore + * @return a valid write IDs list for the input table + */ + public static ValidReaderWriteIdList createValidReaderWriteIdList(TableValidWriteIds tableWriteIds) { + String fullTableName = tableWriteIds.getFullTableName(); + long highWater = tableWriteIds.getWriteIdHighWaterMark(); + List<Long> invalids = tableWriteIds.getInvalidWriteIds(); + BitSet abortedBits = BitSet.valueOf(tableWriteIds.getAbortedBits()); + long[] exceptions = new long[invalids.size()]; + int i = 0; + for (long writeId : invalids) { + exceptions[i++] = writeId; + } + if (tableWriteIds.isSetMinOpenWriteId()) { + return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater, + tableWriteIds.getMinOpenWriteId()); + } else { + return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater); + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index c551b80..6a1cd3b 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -316,7 +316,7 @@ public class HiveAlterHandler implements AlterHandler { } } else { // operations other than table rename - if (MetaStoreUtils.requireCalStats(null, null, newt, environmentContext) && + if (MetaStoreServerUtils.requireCalStats(null, null, newt, environmentContext) && !isPartitionedTable) { Database db = msdb.getDatabase(catName, newDbName); // Update table stats. For partitioned table, we update stats in alterPartition() @@ -467,7 +467,7 @@ public class HiveAlterHandler implements AlterHandler { "Unable to alter partition because table or database does not exist."); } oldPart = msdb.getPartition(catName, dbname, name, new_part.getValues()); - if (MetaStoreUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { + if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { // if stats are same, no need to update if (MetaStoreServerUtils.isFastStatsSame(oldPart, new_part)) { MetaStoreServerUtils.updateBasicState(environmentContext, new_part.getParameters()); @@ -611,7 +611,7 @@ public class HiveAlterHandler implements AlterHandler { new_part.getSd().setLocation(oldPart.getSd().getLocation()); } - if (MetaStoreUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { + if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { MetaStoreServerUtils.updatePartitionStatsFast( new_part, tbl, wh, false, true, environmentContext, false); } @@ -711,7 +711,7 @@ public class HiveAlterHandler implements AlterHandler { oldParts.add(oldTmpPart); partValsList.add(tmpPart.getValues()); - if (MetaStoreUtils.requireCalStats(oldTmpPart, tmpPart, tbl, environmentContext)) { + if (MetaStoreServerUtils.requireCalStats(oldTmpPart, tmpPart, tbl, environmentContext)) { // Check if stats are same, no need to update if (MetaStoreServerUtils.isFastStatsSame(oldTmpPart, tmpPart)) { MetaStoreServerUtils.updateBasicState(environmentContext, tmpPart.getParameters()); http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 6b660f5..33b22a9 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1862,7 +1862,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { } } if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) && - !MetaStoreServerUtils.isView(tbl)) { + !MetaStoreUtils.isView(tbl)) { MetaStoreServerUtils.updateTableStatsSlow(db, tbl, wh, madeDir, false, envContext); } @@ -3804,7 +3804,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { updateStatsTbl = !Boolean.valueOf(tblParams.get(StatsSetupConst.DO_NOT_UPDATE_STATS)); } if (!MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) || - MetaStoreServerUtils.isView(tbl) || + MetaStoreUtils.isView(tbl) || !updateStatsTbl) { return false; } http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index faf6810..a0ff79c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -62,7 +62,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.hooks.URIResolverHook; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.ObjectPair; @@ -2726,19 +2726,19 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { @Override public ValidTxnList getValidTxns() throws TException { - return TxnUtils.createValidReadTxnList(client.get_open_txns(), 0); + return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), 0); } @Override public ValidTxnList getValidTxns(long currentTxn) throws TException { - return TxnUtils.createValidReadTxnList(client.get_open_txns(), currentTxn); + return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), currentTxn); } @Override public ValidWriteIdList getValidWriteIds(String fullTableName) throws TException { GetValidWriteIdsRequest rqst = new GetValidWriteIdsRequest(Collections.singletonList(fullTableName), null); GetValidWriteIdsResponse validWriteIds = client.get_valid_write_ids(rqst); - return TxnUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0)); + return TxnCommonUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0)); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java index 9ce7d6d..d35c960 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsRebuildLockCleanerTask.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.slf4j.Logger; @@ -62,7 +63,7 @@ public class MaterializationsRebuildLockCleanerTask implements MetastoreTaskThre TxnStore.MutexAPI.LockHandle handle = null; try { handle = txnHandler.getMutexAPI().acquireLock(TxnStore.MUTEX_KEY.MaterializationRebuild.name()); - ValidTxnList validTxnList = TxnUtils.createValidReadTxnList(txnHandler.getOpenTxns(), 0); + ValidTxnList validTxnList = TxnCommonUtils.createValidReadTxnList(txnHandler.getOpenTxns(), 0); long removedCnt = txnHandler.cleanupMaterializationRebuildLocks(validTxnList, MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.TXN_TIMEOUT, TimeUnit.MILLISECONDS)); if (removedCnt > 0) { http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java index 8d84fd7..b35dc7c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java @@ -20,13 +20,11 @@ package org.apache.hadoop.hive.metastore.security; import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.util.concurrent.TimeUnit; -import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.utils.SecurityUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.UserGroupInformation; @@ -139,7 +137,7 @@ public class MetastoreDelegationTokenManager { } private DelegationTokenStore getTokenStore(Configuration conf) throws IOException { - String tokenStoreClassName = SecurityUtils.getTokenStoreClassName(conf); + String tokenStoreClassName = MetaStoreServerUtils.getTokenStoreClassName(conf); try { Class<? extends DelegationTokenStore> storeClass = Class.forName(tokenStoreClassName).asSubclass(DelegationTokenStore.class); http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 015dfd9..2df61b4 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -1252,7 +1252,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { validTxnList = new ValidReadTxnList(rqst.getValidTxnList()); } else { // Passing 0 for currentTxn means, this validTxnList is not wrt to any txn - validTxnList = TxnUtils.createValidReadTxnList(getOpenTxns(), 0); + validTxnList = TxnCommonUtils.createValidReadTxnList(getOpenTxns(), 0); } try { /** http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java index bd202ed..3bb1f0c 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java @@ -20,19 +20,16 @@ package org.apache.hadoop.hive.metastore.txn; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ValidCompactorWriteIdList; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; -import org.apache.hadoop.hive.common.ValidReadTxnList; -import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidTxnWriteIdList; -import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.TransactionalValidationListener; -import org.apache.hadoop.hive.metastore.api.*; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -49,101 +46,6 @@ public class TxnUtils { static final public char STAT_OBSOLETE = 's'; /** - * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a - * {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to - * read the files, and thus treats both open and aborted transactions as invalid. - * @param txns txn list from the metastore - * @param currentTxn Current transaction that the user has open. If this is greater than 0 it - * will be removed from the exceptions list so that the user sees his own - * transaction as valid. - * @return a valid txn list. - */ - public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) { - /* - * The highWaterMark should be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0 - * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which - * doesn't make sense for Snapshot Isolation. Of course for Read Committed, the list should - * include the latest committed set. - */ - long highWaterMark = (currentTxn > 0) ? Math.min(currentTxn, txns.getTxn_high_water_mark()) - : txns.getTxn_high_water_mark(); - - // Open txns are already sorted in ascending order. This list may or may not include HWM - // but it is guaranteed that list won't have txn > HWM. But, if we overwrite the HWM with currentTxn - // then need to truncate the exceptions list accordingly. - List<Long> openTxns = txns.getOpen_txns(); - - // We care only about open/aborted txns below currentTxn and hence the size should be determined - // for the exceptions list. The currentTxn will be missing in openTxns list only in rare case like - // txn is aborted by AcidHouseKeeperService and compactor actually cleans up the aborted txns. - // So, for such cases, we get negative value for sizeToHwm with found position for currentTxn, and so, - // we just negate it to get the size. - int sizeToHwm = (currentTxn > 0) ? Collections.binarySearch(openTxns, currentTxn) : openTxns.size(); - sizeToHwm = (sizeToHwm < 0) ? (-sizeToHwm) : sizeToHwm; - long[] exceptions = new long[sizeToHwm]; - BitSet inAbortedBits = BitSet.valueOf(txns.getAbortedBits()); - BitSet outAbortedBits = new BitSet(); - long minOpenTxnId = Long.MAX_VALUE; - int i = 0; - for (long txn : openTxns) { - // For snapshot isolation, we don't care about txns greater than current txn and so stop here. - // Also, we need not include current txn to exceptions list. - if ((currentTxn > 0) && (txn >= currentTxn)) { - break; - } - if (inAbortedBits.get(i)) { - outAbortedBits.set(i); - } else if (minOpenTxnId == Long.MAX_VALUE) { - minOpenTxnId = txn; - } - exceptions[i++] = txn; - } - return new ValidReadTxnList(exceptions, outAbortedBits, highWaterMark, minOpenTxnId); - } - - /** - * Transform a {@link org.apache.hadoop.hive.metastore.api.GetValidWriteIdsResponse} to a - * {@link org.apache.hadoop.hive.common.ValidTxnWriteIdList}. This assumes that the caller intends to - * read the files, and thus treats both open and aborted transactions as invalid. - * @param currentTxnId current txn ID for which we get the valid write ids list - * @param list valid write ids list from the metastore - * @return a valid write IDs list for the whole transaction. - */ - public static ValidTxnWriteIdList createValidTxnWriteIdList(Long currentTxnId, - List<TableValidWriteIds> validIds) { - ValidTxnWriteIdList validTxnWriteIdList = new ValidTxnWriteIdList(currentTxnId); - for (TableValidWriteIds tableWriteIds : validIds) { - validTxnWriteIdList.addTableValidWriteIdList(createValidReaderWriteIdList(tableWriteIds)); - } - return validTxnWriteIdList; - } - - /** - * Transform a {@link org.apache.hadoop.hive.metastore.api.TableValidWriteIds} to a - * {@link org.apache.hadoop.hive.common.ValidReaderWriteIdList}. This assumes that the caller intends to - * read the files, and thus treats both open and aborted write ids as invalid. - * @param tableWriteIds valid write ids for the given table from the metastore - * @return a valid write IDs list for the input table - */ - public static ValidReaderWriteIdList createValidReaderWriteIdList(TableValidWriteIds tableWriteIds) { - String fullTableName = tableWriteIds.getFullTableName(); - long highWater = tableWriteIds.getWriteIdHighWaterMark(); - List<Long> invalids = tableWriteIds.getInvalidWriteIds(); - BitSet abortedBits = BitSet.valueOf(tableWriteIds.getAbortedBits()); - long[] exceptions = new long[invalids.size()]; - int i = 0; - for (long writeId : invalids) { - exceptions[i++] = writeId; - } - if (tableWriteIds.isSetMinOpenWriteId()) { - return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater, - tableWriteIds.getMinOpenWriteId()); - } else { - return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits, highWater); - } - } - - /** * Transform a {@link org.apache.hadoop.hive.metastore.api.TableValidWriteIds} to a * {@link org.apache.hadoop.hive.common.ValidCompactorWriteIdList}. This assumes that the caller intends to * compact the files, and thus treats only open transactions/write ids as invalid. Additionally any http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java index 506d9ab..10ff9df 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java @@ -79,7 +79,10 @@ import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMerger; import org.apache.hadoop.hive.metastore.columnstats.merge.ColumnStatsMergerFactory; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; +import org.apache.hadoop.hive.metastore.security.DBTokenStore; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.apache.hadoop.hive.metastore.security.MemoryTokenStore; +import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore; import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.util.MachineList; @@ -105,6 +108,7 @@ public class MetaStoreServerUtils { return org.apache.commons.lang.StringUtils.defaultString(string); } }; + private static final String DELEGATION_TOKEN_STORE_CLS = "hive.cluster.delegation.token.store.class"; /** * We have a need to sanity-check the map before conversion from persisted objects to @@ -347,13 +351,6 @@ public class MetaStoreServerUtils { return true; } - public static boolean isView(Table table) { - if (table == null) { - return false; - } - return TableType.VIRTUAL_VIEW.toString().equals(table.getTableType()); - } - /** * @param partParams * @return True if the passed Parameters Map contains values for all "Fast Stats". @@ -906,6 +903,80 @@ public class MetaStoreServerUtils { return Character.isLetterOrDigit(c) || c == '_'; } + // check if stats need to be (re)calculated + public static boolean requireCalStats(Partition oldPart, + Partition newPart, Table tbl, + EnvironmentContext environmentContext) { + + if (environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get( + StatsSetupConst.DO_NOT_UPDATE_STATS))) { + return false; + } + + if (MetaStoreUtils.isView(tbl)) { + return false; + } + + if (oldPart == null && newPart == null) { + return true; + } + + // requires to calculate stats if new partition doesn't have it + if ((newPart == null) || (newPart.getParameters() == null) + || !containsAllFastStats(newPart.getParameters())) { + return true; + } + + if (environmentContext != null && environmentContext.isSetProperties()) { + String statsType = environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED); + // no matter STATS_GENERATED is USER or TASK, all need to re-calculate the stats: + // USER: alter table .. update statistics + // TASK: from some sql operation which could collect and compute stats + if (StatsSetupConst.TASK.equals(statsType) || StatsSetupConst.USER.equals(statsType)) { + return true; + } + } + + // requires to calculate stats if new and old have different fast stats + return !isFastStatsSame(oldPart, newPart); + } + + /** + * This method should be used to return the metastore specific tokenstore class name to main + * backwards compatibility + * + * @param conf - HiveConf object + * @return the tokenStoreClass name from the HiveConf. It maps the hive specific tokenstoreclass + * name to metastore module specific class name. For eg: + * hive.cluster.delegation.token.store.class is set to + * org.apache.hadoop.hive.thrift.MemoryTokenStore it returns the equivalent tokenstore + * class defined in the metastore module which is + * org.apache.hadoop.hive.metastore.security.MemoryTokenStore Similarly, + * org.apache.hadoop.hive.thrift.DBTokenStore maps to + * org.apache.hadoop.hive.metastore.security.DBTokenStore and + * org.apache.hadoop.hive.thrift.ZooKeeperTokenStore maps to + * org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore + */ + public static String getTokenStoreClassName(Configuration conf) { + String tokenStoreClass = conf.get(DELEGATION_TOKEN_STORE_CLS, ""); + if (StringUtils.isBlank(tokenStoreClass)) { + // default tokenstore is MemoryTokenStore + return MemoryTokenStore.class.getName(); + } + switch (tokenStoreClass) { + case "org.apache.hadoop.hive.thrift.DBTokenStore": + return DBTokenStore.class.getName(); + case "org.apache.hadoop.hive.thrift.MemoryTokenStore": + return MemoryTokenStore.class.getName(); + case "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore": + return ZooKeeperTokenStore.class.getName(); + default: + return tokenStoreClass; + } + } + // ColumnStatisticsObj with info about its db, table, partition (if table is partitioned) public static class ColStatsObjWithSourceInfo { private final ColumnStatisticsObj colStatsObj; http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 5233bee..c681a87 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -245,46 +245,6 @@ public class MetaStoreUtils { return "TRUE".equalsIgnoreCase(tableParams.get(prop)); } - // check if stats need to be (re)calculated - public static boolean requireCalStats(Partition oldPart, - Partition newPart, Table tbl, - EnvironmentContext environmentContext) { - - if (environmentContext != null - && environmentContext.isSetProperties() - && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get( - StatsSetupConst.DO_NOT_UPDATE_STATS))) { - return false; - } - - if (MetaStoreServerUtils.isView(tbl)) { - return false; - } - - if (oldPart == null && newPart == null) { - return true; - } - - // requires to calculate stats if new partition doesn't have it - if ((newPart == null) || (newPart.getParameters() == null) - || !MetaStoreServerUtils.containsAllFastStats(newPart.getParameters())) { - return true; - } - - if (environmentContext != null && environmentContext.isSetProperties()) { - String statsType = environmentContext.getProperties().get(StatsSetupConst.STATS_GENERATED); - // no matter STATS_GENERATED is USER or TASK, all need to re-calculate the stats: - // USER: alter table .. update statistics - // TASK: from some sql operation which could collect and compute stats - if (StatsSetupConst.TASK.equals(statsType) || StatsSetupConst.USER.equals(statsType)) { - return true; - } - } - - // requires to calculate stats if new and old have different fast stats - return !MetaStoreServerUtils.isFastStatsSame(oldPart, newPart); - } - /** Duplicates AcidUtils; used in a couple places in metastore. */ public static boolean isInsertOnlyTableParam(Map<String, String> params) { @@ -939,4 +899,10 @@ public class MetaStoreUtils { return catName; } + public static boolean isView(Table table) { + if (table == null) { + return false; + } + return TableType.VIRTUAL_VIEW.toString().equals(table.getTableType()); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java index 0b0cfbd..bae1ec3 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java @@ -17,13 +17,8 @@ */ package org.apache.hadoop.hive.metastore.utils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.security.DBTokenStore; import org.apache.hadoop.hive.metastore.security.DelegationTokenIdentifier; import org.apache.hadoop.hive.metastore.security.DelegationTokenSelector; -import org.apache.hadoop.hive.metastore.security.MemoryTokenStore; -import org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -31,31 +26,29 @@ import org.apache.hadoop.security.authentication.util.KerberosUtil; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.security.token.TokenSelector; -import org.apache.zookeeper.client.ZooKeeperSaslClient; - -import javax.security.auth.login.AppConfigurationEntry; import org.apache.thrift.transport.TSSLTransportFactory; import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TSocket; import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; +import org.apache.zookeeper.client.ZooKeeperSaslClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.SSLParameters; import javax.net.ssl.SSLServerSocket; import javax.net.ssl.SSLSocket; -import javax.security.auth.login.LoginException; +import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag; - +import javax.security.auth.login.LoginException; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class SecurityUtils { private static final Logger LOG = LoggerFactory.getLogger(SecurityUtils.class); @@ -192,42 +185,6 @@ public class SecurityUtils { return delegationToken; } - private static final String DELEGATION_TOKEN_STORE_CLS = "hive.cluster.delegation.token.store.class"; - - /** - * This method should be used to return the metastore specific tokenstore class name to main - * backwards compatibility - * - * @param conf - HiveConf object - * @return the tokenStoreClass name from the HiveConf. It maps the hive specific tokenstoreclass - * name to metastore module specific class name. For eg: - * hive.cluster.delegation.token.store.class is set to - * org.apache.hadoop.hive.thrift.MemoryTokenStore it returns the equivalent tokenstore - * class defined in the metastore module which is - * org.apache.hadoop.hive.metastore.security.MemoryTokenStore Similarly, - * org.apache.hadoop.hive.thrift.DBTokenStore maps to - * org.apache.hadoop.hive.metastore.security.DBTokenStore and - * org.apache.hadoop.hive.thrift.ZooKeeperTokenStore maps to - * org.apache.hadoop.hive.metastore.security.ZooKeeperTokenStore - */ - public static String getTokenStoreClassName(Configuration conf) { - String tokenStoreClass = conf.get(DELEGATION_TOKEN_STORE_CLS, ""); - if (StringUtils.isBlank(tokenStoreClass)) { - // default tokenstore is MemoryTokenStore - return MemoryTokenStore.class.getName(); - } - switch (tokenStoreClass) { - case "org.apache.hadoop.hive.thrift.DBTokenStore": - return DBTokenStore.class.getName(); - case "org.apache.hadoop.hive.thrift.MemoryTokenStore": - return MemoryTokenStore.class.getName(); - case "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore": - return ZooKeeperTokenStore.class.getName(); - default: - return tokenStoreClass; - } - } - /** * @return the user name set in hadoop.job.ugi param or the current user from System http://git-wip-us.apache.org/repos/asf/hive/blob/a3f0f99d/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 6e0d9c1..4093aa7 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -62,7 +62,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.hooks.URIResolverHook; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; +import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.ObjectPair; @@ -2187,19 +2187,19 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos @Override public ValidTxnList getValidTxns() throws TException { - return TxnUtils.createValidReadTxnList(client.get_open_txns(), 0); + return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), 0); } @Override public ValidTxnList getValidTxns(long currentTxn) throws TException { - return TxnUtils.createValidReadTxnList(client.get_open_txns(), currentTxn); + return TxnCommonUtils.createValidReadTxnList(client.get_open_txns(), currentTxn); } @Override public ValidWriteIdList getValidWriteIds(String fullTableName) throws TException { GetValidWriteIdsRequest rqst = new GetValidWriteIdsRequest(Collections.singletonList(fullTableName), null); GetValidWriteIdsResponse validWriteIds = client.get_valid_write_ids(rqst); - return TxnUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0)); + return TxnCommonUtils.createValidReaderWriteIdList(validWriteIds.getTblValidWriteIds().get(0)); } @Override
