HIVE-20195 : Split MetastoreUtils into common and server-specific parts (Alexander Kolbasov reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/93779cb1 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/93779cb1 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/93779cb1 Branch: refs/heads/master Commit: 93779cb1010dc357ae3eb9dab77b35b4a444a33c Parents: faf2914 Author: Alexander Kolbasov <[email protected]> Authored: Tue Aug 14 11:04:18 2018 -0700 Committer: Vihang Karajgaonkar <[email protected]> Committed: Tue Aug 14 11:04:54 2018 -0700 ---------------------------------------------------------------------- .../listener/DummyRawStoreFailEvent.java | 2 +- .../AbstractTestAuthorizationApiAuthorizer.java | 2 - .../apache/hadoop/hive/ql/metadata/Hive.java | 5 +- .../apache/hadoop/hive/ql/metadata/Table.java | 3 +- .../hadoop/hive/ql/stats/BasicStatsTask.java | 5 +- .../hadoop/hive/metastore/HiveAlterHandler.java | 25 +- .../hadoop/hive/metastore/HiveMetaStore.java | 53 +- .../hive/metastore/MetaStoreDirectSql.java | 13 +- .../hadoop/hive/metastore/ObjectStore.java | 19 +- .../apache/hadoop/hive/metastore/RawStore.java | 2 +- .../hive/metastore/cache/CachedStore.java | 5 +- .../hive/metastore/cache/SharedCache.java | 8 +- .../aggr/BinaryColumnStatsAggregator.java | 2 +- .../aggr/BooleanColumnStatsAggregator.java | 2 +- .../columnstats/aggr/ColumnStatsAggregator.java | 2 +- .../aggr/DateColumnStatsAggregator.java | 2 +- .../aggr/DecimalColumnStatsAggregator.java | 31 +- .../aggr/DoubleColumnStatsAggregator.java | 2 +- .../aggr/LongColumnStatsAggregator.java | 3 +- .../aggr/StringColumnStatsAggregator.java | 3 +- .../hadoop/hive/metastore/txn/TxnHandler.java | 4 +- .../metastore/utils/HiveStrictManagedUtils.java | 6 +- .../metastore/utils/MetaStoreServerUtils.java | 939 +++++++++++++++++++ .../hive/metastore/utils/MetaStoreUtils.java | 904 +----------------- .../DummyRawStoreControlledCommit.java | 2 +- .../DummyRawStoreForJdoConnection.java | 4 +- .../hive/metastore/TestHiveMetaStore.java | 5 +- .../metastore/TestHiveMetaStoreGetMetaConf.java | 4 +- .../metastore/utils/TestMetaStoreUtils.java | 20 +- 29 files changed, 1057 insertions(+), 1020 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java ---------------------------------------------------------------------- diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java index be40395..0ad2a24 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java @@ -91,7 +91,7 @@ import org.apache.hadoop.hive.metastore.api.WMPool; import org.apache.hadoop.hive.metastore.api.WMNullablePool; import org.apache.hadoop.hive.metastore.api.WriteEventInfo; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.apache.thrift.TException; /** http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java index 69692d0..c10060f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java @@ -31,8 +31,6 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.Role; -import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.security.authorization.MetaStoreAuthzAPIAuthorizerEmbedOnly; import org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener; import org.apache.thrift.TException; http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 45610e3..f846e93 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -104,6 +104,7 @@ import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.ReplChangeManager; import org.apache.hadoop.hive.metastore.api.*; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.AbstractFileMergeOperator; @@ -1957,10 +1958,10 @@ public class Hive { newPartPath, -1, newPartPath.getFileSystem(conf)); } if (filesForStats != null) { - MetaStoreUtils.populateQuickStats(filesForStats, newTPart.getParameters()); + MetaStoreServerUtils.populateQuickStats(filesForStats, newTPart.getParameters()); } else { // The ACID state is probably absent. Warning is logged in the get method. - MetaStoreUtils.clearQuickStats(newTPart.getParameters()); + MetaStoreServerUtils.clearQuickStats(newTPart.getParameters()); } try { LOG.debug("Adding new partition " + newTPart.getSpec()); http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index 9e0cea7..26f21cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -50,6 +50,7 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils; @@ -1073,7 +1074,7 @@ public class Table implements Serializable { } private static String normalize(String colName) throws HiveException { - if (!MetaStoreUtils.validateColumnName(colName)) { + if (!MetaStoreServerUtils.validateColumnName(colName)) { throw new HiveException("Invalid column name '" + colName + "' in the table definition"); } http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java index b9b4a44..6eb1ca2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/BasicStatsTask.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -145,7 +146,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor { : partish.getPartition().getSpec().toString(); LOG.warn("Partition/partfiles is null for: " + spec); if (isMissingAcidState) { - MetaStoreUtils.clearQuickStats(parameters); + MetaStoreServerUtils.clearQuickStats(parameters); return p.getOutput(); } return null; @@ -159,7 +160,7 @@ public class BasicStatsTask implements Serializable, IStatsProcessor { StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); } - MetaStoreUtils.populateQuickStats(partfileStatus, parameters); + MetaStoreServerUtils.populateQuickStats(partfileStatus, parameters); if (statsAggregator != null) { // Update stats for transactional tables (MM, or full ACID with overwrite), even http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 69f6ed5..0441a33 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.messaging.EventMessage; import org.apache.hadoop.hive.metastore.utils.FileUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,7 +111,7 @@ public class HiveAlterHandler implements AlterHandler { if (!MetaStoreUtils.validateName(newTblName, handler.getConf())) { throw new InvalidOperationException(newTblName + " is not a valid object name"); } - String validate = MetaStoreUtils.validateTblColumns(newt.getSd().getCols()); + String validate = MetaStoreServerUtils.validateTblColumns(newt.getSd().getCols()); if (validate != null) { throw new InvalidOperationException("Invalid column " + validate); } @@ -318,12 +319,12 @@ public class HiveAlterHandler implements AlterHandler { !isPartitionedTable) { Database db = msdb.getDatabase(catName, newDbName); // Update table stats. For partitioned table, we update stats in alterPartition() - MetaStoreUtils.updateTableStatsSlow(db, newt, wh, false, true, environmentContext); + MetaStoreServerUtils.updateTableStatsSlow(db, newt, wh, false, true, environmentContext); } if (isPartitionedTable) { //Currently only column related changes can be cascaded in alter table - if(!MetaStoreUtils.areSameColumns(oldt.getSd().getCols(), newt.getSd().getCols())) { + if(!MetaStoreServerUtils.areSameColumns(oldt.getSd().getCols(), newt.getSd().getCols())) { parts = msdb.getPartitions(catName, dbname, name, -1); for (Partition part : parts) { Partition oldPart = new Partition(part); @@ -467,10 +468,10 @@ public class HiveAlterHandler implements AlterHandler { oldPart = msdb.getPartition(catName, dbname, name, new_part.getValues()); if (MetaStoreUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { // if stats are same, no need to update - if (MetaStoreUtils.isFastStatsSame(oldPart, new_part)) { - MetaStoreUtils.updateBasicState(environmentContext, new_part.getParameters()); + if (MetaStoreServerUtils.isFastStatsSame(oldPart, new_part)) { + MetaStoreServerUtils.updateBasicState(environmentContext, new_part.getParameters()); } else { - MetaStoreUtils.updatePartitionStatsFast( + MetaStoreServerUtils.updatePartitionStatsFast( new_part, tbl, wh, false, true, environmentContext, false); } } @@ -610,7 +611,7 @@ public class HiveAlterHandler implements AlterHandler { } if (MetaStoreUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { - MetaStoreUtils.updatePartitionStatsFast( + MetaStoreServerUtils.updatePartitionStatsFast( new_part, tbl, wh, false, true, environmentContext, false); } @@ -711,10 +712,10 @@ public class HiveAlterHandler implements AlterHandler { if (MetaStoreUtils.requireCalStats(oldTmpPart, tmpPart, tbl, environmentContext)) { // Check if stats are same, no need to update - if (MetaStoreUtils.isFastStatsSame(oldTmpPart, tmpPart)) { - MetaStoreUtils.updateBasicState(environmentContext, tmpPart.getParameters()); + if (MetaStoreServerUtils.isFastStatsSame(oldTmpPart, tmpPart)) { + MetaStoreServerUtils.updateBasicState(environmentContext, tmpPart.getParameters()); } else { - MetaStoreUtils.updatePartitionStatsFast( + MetaStoreServerUtils.updatePartitionStatsFast( tmpPart, tbl, wh, false, true, environmentContext, false); } } @@ -810,7 +811,7 @@ public class HiveAlterHandler implements AlterHandler { List<ColumnStatisticsObj> newStatsObjs = new ArrayList<>(); ColumnStatistics colStats = null; boolean updateColumnStats = !newDbName.equals(dbName) || !newTableName.equals(tableName) - || !MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols); + || !MetaStoreServerUtils.columnsIncludedByNameType(oldCols, newCols); if (updateColumnStats) { List<String> oldColNames = new ArrayList<>(oldCols.size()); for (FieldSchema oldCol : oldCols) { @@ -886,7 +887,7 @@ public class HiveAlterHandler implements AlterHandler { || !oldPartName.equals(newPartName); // do not need to update column stats if alter partition is not for rename or changing existing columns - if (!rename && MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols)) { + if (!rename && MetaStoreServerUtils.columnsIncludedByNameType(oldCols, newCols)) { return newPartsColStats; } List<String> oldColNames = new ArrayList<>(oldCols.size()); http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index d34e944..95b08eb 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -152,6 +152,7 @@ import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager import org.apache.hadoop.hive.metastore.security.TUGIContainingTransport; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.hive.metastore.utils.CommonCliOptions; import org.apache.hadoop.hive.metastore.utils.FileUtils; @@ -521,7 +522,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { @Override public void init() throws MetaException { - initListeners = MetaStoreUtils.getMetaStoreListeners( + initListeners = MetaStoreServerUtils.getMetaStoreListeners( MetaStoreInitListener.class, conf, MetastoreConf.getVar(conf, ConfVars.INIT_HOOKS)); for (MetaStoreInitListener singleInitListener: initListeners) { MetaStoreInitContext context = new MetaStoreInitContext(); @@ -553,20 +554,20 @@ public class HiveMetaStore extends ThriftHiveMetastore { } - preListeners = MetaStoreUtils.getMetaStoreListeners(MetaStorePreEventListener.class, + preListeners = MetaStoreServerUtils.getMetaStoreListeners(MetaStorePreEventListener.class, conf, MetastoreConf.getVar(conf, ConfVars.PRE_EVENT_LISTENERS)); preListeners.add(0, new TransactionalValidationListener(conf)); - listeners = MetaStoreUtils.getMetaStoreListeners(MetaStoreEventListener.class, conf, + listeners = MetaStoreServerUtils.getMetaStoreListeners(MetaStoreEventListener.class, conf, MetastoreConf.getVar(conf, ConfVars.EVENT_LISTENERS)); listeners.add(new SessionPropertiesListener(conf)); listeners.add(new AcidEventListener(conf)); - transactionalListeners = MetaStoreUtils.getMetaStoreListeners(TransactionalMetaStoreEventListener.class, + transactionalListeners = MetaStoreServerUtils.getMetaStoreListeners(TransactionalMetaStoreEventListener.class, conf, MetastoreConf.getVar(conf, ConfVars.TRANSACTIONAL_EVENT_LISTENERS)); if (Metrics.getRegistry() != null) { listeners.add(new HMSMetricsListener(conf)); } - endFunctionListeners = MetaStoreUtils.getMetaStoreListeners( + endFunctionListeners = MetaStoreServerUtils.getMetaStoreListeners( MetaStoreEndFunctionListener.class, conf, MetastoreConf.getVar(conf, ConfVars.END_FUNCTION_LISTENERS)); String partitionValidationRegex = @@ -1794,23 +1795,23 @@ public class HiveMetaStore extends ThriftHiveMetastore { throw new InvalidObjectException(tbl.getTableName() + " is not a valid object name"); } - String validate = MetaStoreUtils.validateTblColumns(tbl.getSd().getCols()); + String validate = MetaStoreServerUtils.validateTblColumns(tbl.getSd().getCols()); if (validate != null) { throw new InvalidObjectException("Invalid column " + validate); } if (tbl.getPartitionKeys() != null) { - validate = MetaStoreUtils.validateTblColumns(tbl.getPartitionKeys()); + validate = MetaStoreServerUtils.validateTblColumns(tbl.getPartitionKeys()); if (validate != null) { throw new InvalidObjectException("Invalid partition column " + validate); } } SkewedInfo skew = tbl.getSd().getSkewedInfo(); if (skew != null) { - validate = MetaStoreUtils.validateSkewedColNames(skew.getSkewedColNames()); + validate = MetaStoreServerUtils.validateSkewedColNames(skew.getSkewedColNames()); if (validate != null) { throw new InvalidObjectException("Invalid skew column " + validate); } - validate = MetaStoreUtils.validateSkewedColNamesSubsetCol( + validate = MetaStoreServerUtils.validateSkewedColNamesSubsetCol( skew.getSkewedColNames(), tbl.getSd().getCols()); if (validate != null) { throw new InvalidObjectException("Invalid skew column " + validate); @@ -1861,8 +1862,8 @@ public class HiveMetaStore extends ThriftHiveMetastore { } } if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) && - !MetaStoreUtils.isView(tbl)) { - MetaStoreUtils.updateTableStatsSlow(db, tbl, wh, madeDir, false, envContext); + !MetaStoreServerUtils.isView(tbl)) { + MetaStoreServerUtils.updateTableStatsSlow(db, tbl, wh, madeDir, false, envContext); } // set create time @@ -3127,7 +3128,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { part.setTableName(tableName); part.setValues(part_vals); - MetaStoreUtils.validatePartitionNameCharacters(part_vals, partitionValidationPattern); + MetaStoreServerUtils.validatePartitionNameCharacters(part_vals, partitionValidationPattern); tbl = ms.getTable(part.getCatName(), part.getDbName(), part.getTableName(), null); if (tbl == null) { @@ -3174,8 +3175,8 @@ public class HiveMetaStore extends ThriftHiveMetastore { part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(time)); if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) && - !MetaStoreUtils.isView(tbl)) { - MetaStoreUtils.updatePartitionStatsFast(part, tbl, wh, madeDir, false, envContext, true); + !MetaStoreServerUtils.isView(tbl)) { + MetaStoreServerUtils.updatePartitionStatsFast(part, tbl, wh, madeDir, false, envContext, true); } if (ms.addPartition(part)) { @@ -3734,7 +3735,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { private boolean startAddPartition( RawStore ms, Partition part, List<FieldSchema> partitionKeys, boolean ifNotExists) throws TException { - MetaStoreUtils.validatePartitionNameCharacters(part.getValues(), + MetaStoreServerUtils.validatePartitionNameCharacters(part.getValues(), partitionValidationPattern); boolean doesExist = ms.doesPartitionExist(part.getCatName(), part.getDbName(), part.getTableName(), partitionKeys, part.getValues()); @@ -3798,8 +3799,8 @@ public class HiveMetaStore extends ThriftHiveMetastore { private void initializeAddedPartition( final Table tbl, final PartitionSpecProxy.PartitionIterator part, boolean madeDir) throws MetaException { if (MetastoreConf.getBoolVar(conf, ConfVars.STATS_AUTO_GATHER) && - !MetaStoreUtils.isView(tbl)) { - MetaStoreUtils.updatePartitionStatsFast(part, tbl, wh, madeDir, false, null, true); + !MetaStoreServerUtils.isView(tbl)) { + MetaStoreServerUtils.updatePartitionStatsFast(part, tbl, wh, madeDir, false, null, true); } // set create time @@ -4883,7 +4884,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { try { firePreEvent(new PreAlterPartitionEvent(db_name, tbl_name, part_vals, new_part, this)); if (part_vals != null && !part_vals.isEmpty()) { - MetaStoreUtils.validatePartitionNameCharacters(new_part.getValues(), + MetaStoreServerUtils.validatePartitionNameCharacters(new_part.getValues(), partitionValidationPattern); } @@ -7011,10 +7012,10 @@ public class HiveMetaStore extends ThriftHiveMetastore { Exception ex = null; try { if (throw_exception) { - MetaStoreUtils.validatePartitionNameCharacters(part_vals, partitionValidationPattern); + MetaStoreServerUtils.validatePartitionNameCharacters(part_vals, partitionValidationPattern); ret = true; } else { - ret = MetaStoreUtils.partitionNameHasValidCharacters(part_vals, + ret = MetaStoreServerUtils.partitionNameHasValidCharacters(part_vals, partitionValidationPattern); } } catch (MetaException e) { @@ -7610,7 +7611,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { for (ColumnStatistics csNew : csNews) { String partName = csNew.getStatsDesc().getPartName(); if (newStatsMap.containsKey(partName)) { - MetaStoreUtils.mergeColStats(csNew, newStatsMap.get(partName)); + MetaStoreServerUtils.mergeColStats(csNew, newStatsMap.get(partName)); } newStatsMap.put(partName, csNew); } @@ -7670,10 +7671,10 @@ public class HiveMetaStore extends ThriftHiveMetastore { csNew.setStatsObj(Lists.newArrayList()); } else { // we first use getParameters() to prune the stats - MetaStoreUtils.getMergableCols(csNew, part.getParameters()); + MetaStoreServerUtils.getMergableCols(csNew, part.getParameters()); // we merge those that can be merged if (csOld != null && csOld.getStatsObjSize() != 0 && !csNew.getStatsObj().isEmpty()) { - MetaStoreUtils.mergeColStats(csNew, csOld); + MetaStoreServerUtils.mergeColStats(csNew, csOld); } } @@ -7726,11 +7727,11 @@ public class HiveMetaStore extends ThriftHiveMetastore { firstColStats.setStatsObj(Lists.newArrayList()); } else { Table t = getTable(catName, dbName, tableName); - MetaStoreUtils.getMergableCols(firstColStats, t.getParameters()); + MetaStoreServerUtils.getMergableCols(firstColStats, t.getParameters()); // we merge those that can be merged if (csOld != null && csOld.getStatsObjSize() != 0 && !firstColStats.getStatsObj().isEmpty()) { - MetaStoreUtils.mergeColStats(firstColStats, csOld); + MetaStoreServerUtils.mergeColStats(firstColStats, csOld); } } @@ -7835,7 +7836,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { LOG.error("Cannot obtain username", ex); throw ex; } - if (!MetaStoreUtils.checkUserHasHostProxyPrivileges(user, conf, getIPAddress())) { + if (!MetaStoreServerUtils.checkUserHasHostProxyPrivileges(user, conf, getIPAddress())) { throw new MetaException("User " + user + " is not allowed to perform this API call"); } } http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 95d9fe2..571c789 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -89,8 +89,9 @@ import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.apache.hive.common.util.BloomFilter; import org.datanucleus.store.rdbms.query.ForwardQueryResult; import org.slf4j.Logger; @@ -384,7 +385,7 @@ class MetaStoreDirectSql { db.setOwnerType( (null == type || type.trim().isEmpty()) ? null : PrincipalType.valueOf(type)); db.setCatalogName(extractSqlString(dbline[6])); - db.setParameters(MetaStoreUtils.trimMapNulls(dbParams,convertMapNullsToEmptyStrings)); + db.setParameters(MetaStoreServerUtils.trimMapNulls(dbParams,convertMapNullsToEmptyStrings)); if (LOG.isDebugEnabled()){ LOG.debug("getDatabase: directsql returning db " + db.getName() + " locn["+db.getLocationUri() +"] desc [" +db.getDescription() @@ -772,7 +773,7 @@ class MetaStoreDirectSql { }}); // Perform conversion of null map values for (Partition t : partitions.values()) { - t.setParameters(MetaStoreUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); + t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } queryText = "select \"PART_ID\", \"PART_KEY_VAL\" from " + PARTITION_KEY_VALS + "" @@ -805,7 +806,7 @@ class MetaStoreDirectSql { }}); // Perform conversion of null map values for (StorageDescriptor t : sds.values()) { - t.setParameters(MetaStoreUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); + t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } queryText = "select \"SD_ID\", \"COLUMN_NAME\", " + SORT_COLS + ".\"ORDER\"" @@ -947,7 +948,7 @@ class MetaStoreDirectSql { }}); // Perform conversion of null map values for (SerDeInfo t : serdes.values()) { - t.setParameters(MetaStoreUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); + t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings)); } return orderedResult; @@ -1615,7 +1616,7 @@ class MetaStoreDirectSql { List<ColumnStatistics> partStats = getPartitionStats(catName, dbName, tableName, partNames, colNames, true); // 2. use util function to aggr stats - return MetaStoreUtils.aggrPartitionStats(partStats, catName, dbName, tableName, partNames, colNames, + return MetaStoreServerUtils.aggrPartitionStats(partStats, catName, dbName, tableName, partNames, colNames, areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner); } http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 8af164e..8e2f94e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -96,13 +96,12 @@ import org.apache.hadoop.hive.metastore.parser.ExpressionTree; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.tools.SQLGenerator; -import org.apache.hadoop.hive.metastore.txn.TxnDbUtil; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.ObjectPair; -import org.apache.hive.common.util.TxnIdUtils; import org.apache.thrift.TException; import org.datanucleus.AbstractNucleusContext; import org.datanucleus.ClassLoaderResolver; @@ -1899,7 +1898,7 @@ public class ObjectStore implements RawStore, Configurable { /** Makes shallow copy of a map to avoid DataNucleus mucking with our objects. */ private Map<String, String> convertMap(Map<String, String> dnMap) { - return MetaStoreUtils.trimMapNulls(dnMap, + return MetaStoreServerUtils.trimMapNulls(dnMap, MetastoreConf.getBoolVar(getConf(), ConfVars.ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS)); } @@ -2296,7 +2295,7 @@ public class ObjectStore implements RawStore, Configurable { private boolean isValidPartition( Partition part, List<FieldSchema> partitionKeys, boolean ifNotExists) throws MetaException { - MetaStoreUtils.validatePartitionNameCharacters(part.getValues(), + MetaStoreServerUtils.validatePartitionNameCharacters(part.getValues(), partitionValidationPattern); boolean doesExist = doesPartitionExist(part.getCatName(), part.getDbName(), part.getTableName(), partitionKeys, part.getValues()); @@ -8889,21 +8888,21 @@ public class ObjectStore implements RawStore, Configurable { } @Override - public List<MetaStoreUtils.ColStatsObjWithSourceInfo> getPartitionColStatsForDatabase(String catName, String dbName) + public List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getPartitionColStatsForDatabase(String catName, String dbName) throws MetaException, NoSuchObjectException { final boolean enableBitVector = MetastoreConf.getBoolVar(getConf(), ConfVars.STATS_FETCH_BITVECTOR); - return new GetHelper<List<MetaStoreUtils.ColStatsObjWithSourceInfo>>( + return new GetHelper<List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>>( catName, dbName, null, true, false) { @Override - protected List<MetaStoreUtils.ColStatsObjWithSourceInfo> getSqlResult( - GetHelper<List<MetaStoreUtils.ColStatsObjWithSourceInfo>> ctx) throws MetaException { + protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getSqlResult( + GetHelper<List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>> ctx) throws MetaException { return directSql.getColStatsForAllTablePartitions(catName, dbName, enableBitVector); } @Override - protected List<MetaStoreUtils.ColStatsObjWithSourceInfo> getJdoResult( - GetHelper<List<MetaStoreUtils.ColStatsObjWithSourceInfo>> ctx) + protected List<MetaStoreServerUtils.ColStatsObjWithSourceInfo> getJdoResult( + GetHelper<List<MetaStoreServerUtils.ColStatsObjWithSourceInfo>> ctx) throws MetaException, NoSuchObjectException { // This is fast path for query optimizations, if we can find this info // quickly using directSql, do it. No point in failing back to slow path http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java index 8d647a0..b61ee81 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -32,7 +32,7 @@ import java.util.Map; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.apache.thrift.TException; public interface RawStore extends Configurable { http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java index 1d53244..0445cbf 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java @@ -59,8 +59,9 @@ import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.apache.hadoop.hive.metastore.utils.StringUtils; import org.apache.thrift.TException; import org.slf4j.Logger; @@ -1902,7 +1903,7 @@ public class CachedStore implements RawStore, Configurable { } // Note that enableBitVector does not apply here because ColumnStatisticsObj // itself will tell whether bitvector is null or not and aggr logic can automatically apply. - return new MergedColumnStatsForPartitions(MetaStoreUtils.aggrPartitionStats(colStatsMap, + return new MergedColumnStatsForPartitions(MetaStoreServerUtils.aggrPartitionStats(colStatsMap, partNames, areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner), partsFound); } http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index 24f940c..37c300e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -47,7 +47,7 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableMeta; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.StringUtils; import org.apache.hadoop.hive.ql.util.IncrementalObjectSizeEstimator; import org.apache.hadoop.hive.ql.util.IncrementalObjectSizeEstimator.ObjectEstimator; @@ -709,7 +709,7 @@ public class SharedCache { } setTable(tblCopy); if (tblCopy.getSd() != null) { - sdHash = MetaStoreUtils.hashStorageDescriptor(tblCopy.getSd(), md); + sdHash = MetaStoreServerUtils.hashStorageDescriptor(tblCopy.getSd(), md); StorageDescriptor sd = tblCopy.getSd(); sharedCache.increSd(sd, sdHash); tblCopy.setSd(null); @@ -727,7 +727,7 @@ public class SharedCache { Partition partCopy = part.deepCopy(); PartitionWrapper wrapper; if (part.getSd() != null) { - byte[] sdHash = MetaStoreUtils.hashStorageDescriptor(part.getSd(), md); + byte[] sdHash = MetaStoreServerUtils.hashStorageDescriptor(part.getSd(), md); StorageDescriptor sd = part.getSd(); sharedCache.increSd(sd, sdHash); partCopy.setSd(null); @@ -1146,7 +1146,7 @@ public class SharedCache { } } if (tbl.getSd() != null) { - byte[] sdHash = MetaStoreUtils.hashStorageDescriptor(tbl.getSd(), md); + byte[] sdHash = MetaStoreServerUtils.hashStorageDescriptor(tbl.getSd(), md); StorageDescriptor sd = tbl.getSd(); increSd(sd, sdHash); tblCopy.setSd(null); http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java index c18b4c7..c885cf2 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BinaryColumnStatsAggregator.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; public class BinaryColumnStatsAggregator extends ColumnStatsAggregator { http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java index 7630183..6fafab5 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/BooleanColumnStatsAggregator.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; public class BooleanColumnStatsAggregator extends ColumnStatsAggregator { http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java index 0beaf60..c432576 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/ColumnStatsAggregator.java @@ -23,7 +23,7 @@ import java.util.List; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; public abstract class ColumnStatsAggregator { public boolean useDensityFunctionForNDVEstimation; http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java index 9495424..ddc1455 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hive.metastore.api.Date; import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java index 8739e73..cc18dab 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java @@ -28,15 +28,14 @@ import java.util.Map; import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory; -import org.apache.hadoop.hive.metastore.StatObjectConverter; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.utils.DecimalUtils; import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,7 +102,7 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen DecimalColumnStatsDataInspector newData = decimalInspectorFromStats(cso); lowerBound = Math.max(lowerBound, newData.getNumDVs()); higherBound += newData.getNumDVs(); - densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils + densityAvgSum += (MetaStoreServerUtils.decimalToDouble(newData.getHighValue()) - MetaStoreServerUtils .decimalToDouble(newData.getLowValue())) / newData.getNumDVs(); if (ndvEstimator != null) { ndvEstimator.mergeEstimators(newData.getNdvEstimator()); @@ -111,13 +110,13 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen if (aggregateData == null) { aggregateData = newData.deepCopy(); } else { - if (MetaStoreUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreUtils + if (MetaStoreServerUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreServerUtils .decimalToDouble(newData.getLowValue())) { aggregateData.setLowValue(aggregateData.getLowValue()); } else { aggregateData.setLowValue(newData.getLowValue()); } - if (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreUtils + if (MetaStoreServerUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreServerUtils .decimalToDouble(newData.getHighValue())) { aggregateData.setHighValue(aggregateData.getHighValue()); } else { @@ -138,7 +137,7 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen // We have estimation, lowerbound and higherbound. We use estimation // if it is between lowerbound and higherbound. double densityAvg = densityAvgSum / partNames.size(); - estimation = (long) ((MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils + estimation = (long) ((MetaStoreServerUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreServerUtils .decimalToDouble(aggregateData.getLowValue())) / densityAvg); if (estimation < lowerBound) { estimation = lowerBound; @@ -171,7 +170,7 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen String partName = csp.getPartName(); DecimalColumnStatsData newData = cso.getStatsData().getDecimalStats(); if (useDensityFunctionForNDVEstimation) { - densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils + densityAvgSum += (MetaStoreServerUtils.decimalToDouble(newData.getHighValue()) - MetaStoreServerUtils .decimalToDouble(newData.getLowValue())) / newData.getNumDVs(); } adjustedIndexMap.put(partName, (double) indexMap.get(partName)); @@ -201,7 +200,7 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen csd.setDecimalStats(aggregateData); adjustedStatsMap.put(pseudoPartName.toString(), csd); if (useDensityFunctionForNDVEstimation) { - densityAvgSum += (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils + densityAvgSum += (MetaStoreServerUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreServerUtils .decimalToDouble(aggregateData.getLowValue())) / aggregateData.getNumDVs(); } // reset everything @@ -220,13 +219,13 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen if (aggregateData == null) { aggregateData = newData.deepCopy(); } else { - if (MetaStoreUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreUtils + if (MetaStoreServerUtils.decimalToDouble(aggregateData.getLowValue()) < MetaStoreServerUtils .decimalToDouble(newData.getLowValue())) { aggregateData.setLowValue(aggregateData.getLowValue()); } else { aggregateData.setLowValue(newData.getLowValue()); } - if (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreUtils + if (MetaStoreServerUtils.decimalToDouble(aggregateData.getHighValue()) > MetaStoreServerUtils .decimalToDouble(newData.getHighValue())) { aggregateData.setHighValue(aggregateData.getHighValue()); } else { @@ -244,7 +243,7 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen csd.setDecimalStats(aggregateData); adjustedStatsMap.put(pseudoPartName.toString(), csd); if (useDensityFunctionForNDVEstimation) { - densityAvgSum += (MetaStoreUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreUtils + densityAvgSum += (MetaStoreServerUtils.decimalToDouble(aggregateData.getHighValue()) - MetaStoreServerUtils .decimalToDouble(aggregateData.getLowValue())) / aggregateData.getNumDVs(); } } @@ -283,8 +282,8 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen double minInd = adjustedIndexMap.get(list.get(0).getKey()); double maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey()); double lowValue = 0; - double min = MetaStoreUtils.decimalToDouble(list.get(0).getValue().getLowValue()); - double max = MetaStoreUtils.decimalToDouble(list.get(list.size() - 1).getValue().getLowValue()); + double min = MetaStoreServerUtils.decimalToDouble(list.get(0).getValue().getLowValue()); + double max = MetaStoreServerUtils.decimalToDouble(list.get(list.size() - 1).getValue().getLowValue()); if (minInd == maxInd) { lowValue = min; } else if (minInd < maxInd) { @@ -306,8 +305,8 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen minInd = adjustedIndexMap.get(list.get(0).getKey()); maxInd = adjustedIndexMap.get(list.get(list.size() - 1).getKey()); double highValue = 0; - min = MetaStoreUtils.decimalToDouble(list.get(0).getValue().getHighValue()); - max = MetaStoreUtils.decimalToDouble(list.get(list.size() - 1).getValue().getHighValue()); + min = MetaStoreServerUtils.decimalToDouble(list.get(0).getValue().getHighValue()); + max = MetaStoreServerUtils.decimalToDouble(list.get(list.size() - 1).getValue().getHighValue()); if (minInd == maxInd) { highValue = min; } else if (minInd < maxInd) { http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java index 5ad8453..7c217fc 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java index ab31539..2b237e5 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java @@ -28,13 +28,12 @@ import java.util.Map; import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java index 92fdda5..6fb0fb5 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java @@ -28,13 +28,12 @@ import java.util.Map; import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils.ColStatsObjWithSourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 8edc387..015dfd9 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -80,7 +80,7 @@ import org.apache.hadoop.hive.metastore.metrics.Metrics; import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; import org.apache.hadoop.hive.metastore.tools.SQLGenerator; import org.apache.hadoop.hive.metastore.utils.JavaUtils; -import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.StringableMap; import org.apache.hadoop.util.StringUtils; import org.slf4j.Logger; @@ -329,7 +329,7 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { maxOpenTxns = MetastoreConf.getIntVar(conf, ConfVars.MAX_OPEN_TXNS); try { - transactionalListeners = MetaStoreUtils.getMetaStoreListeners( + transactionalListeners = MetaStoreServerUtils.getMetaStoreListeners( TransactionalMetaStoreEventListener.class, conf, MetastoreConf.getVar(conf, ConfVars.TRANSACTIONAL_EVENT_LISTENERS)); } catch(MetaException e) { http://git-wip-us.apache.org/repos/asf/hive/blob/93779cb1/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java index fac3c22..7213f8e 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/HiveStrictManagedUtils.java @@ -18,10 +18,6 @@ package org.apache.hadoop.hive.metastore.utils; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -54,7 +50,7 @@ public class HiveStrictManagedUtils { TableType tableType = TableType.valueOf(table.getTableType()); if (tableType == TableType.MANAGED_TABLE) { - if (!MetaStoreUtils.isTransactionalTable(table.getParameters())) { + if (!MetaStoreServerUtils.isTransactionalTable(table.getParameters())) { return createValidationError(table, "Table is marked as a managed table but is not transactional."); } if (MetaStoreUtils.isNonNativeTable(table)) {
