Repository: hive Updated Branches: refs/heads/master fbb3ed15f -> 456a65180
http://git-wip-us.apache.org/repos/asf/hive/blob/456a6518/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java index d12cdc0..e6823d3 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,27 +44,24 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class); @Override - public ColumnStatisticsObj aggregate(String colName, List<String> partNames, - List<ColumnStatistics> css) throws MetaException { + public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo, + List<String> partNames, boolean areAllPartsFound) throws MetaException { ColumnStatisticsObj statsObj = null; - + String colType = null; + String colName = null; // check if all the ColumnStatisticsObjs contain stats and all the ndv are // bitvectors - boolean doAllPartitionContainStats = partNames.size() == css.size(); - LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats); + boolean doAllPartitionContainStats = partNames.size() == colStatsWithSourceInfo.size(); NumDistinctValueEstimator ndvEstimator = null; - String colType = null; - for (ColumnStatistics cs : css) { - if (cs.getStatsObjSize() != 1) { - throw new MetaException( - "The number of columns should be exactly one in aggrStats, but found " - + cs.getStatsObjSize()); - } - ColumnStatisticsObj cso = cs.getStatsObjIterator().next(); + for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { + ColumnStatisticsObj cso = csp.getColStatsObj(); if (statsObj == null) { + colName = cso.getColName(); colType = cso.getColType(); - statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso - .getStatsData().getSetField()); + statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, + cso.getStatsData().getSetField()); + LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, + doAllPartitionContainStats); } LongColumnStatsDataInspector longColumnStatsData = (LongColumnStatsDataInspector) cso.getStatsData().getLongStats(); @@ -91,13 +89,13 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements } LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null)); ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData(); - if (doAllPartitionContainStats || css.size() < 2) { + if (doAllPartitionContainStats || colStatsWithSourceInfo.size() < 2) { LongColumnStatsDataInspector aggregateData = null; long lowerBound = 0; long higherBound = 0; double densityAvgSum = 0.0; - for (ColumnStatistics cs : css) { - ColumnStatisticsObj cso = cs.getStatsObjIterator().next(); + for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { + ColumnStatisticsObj cso = csp.getColStatsObj(); LongColumnStatsDataInspector newData = (LongColumnStatsDataInspector) cso.getStatsData().getLongStats(); lowerBound = Math.max(lowerBound, newData.getNumDVs()); @@ -155,9 +153,9 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements if (ndvEstimator == null) { // if not every partition uses bitvector for ndv, we just fall back to // the traditional extrapolation methods. - for (ColumnStatistics cs : css) { - String partName = cs.getStatsDesc().getPartName(); - ColumnStatisticsObj cso = cs.getStatsObjIterator().next(); + for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { + ColumnStatisticsObj cso = csp.getColStatsObj(); + String partName = csp.getPartName(); LongColumnStatsData newData = cso.getStatsData().getLongStats(); if (useDensityFunctionForNDVEstimation) { densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs(); @@ -173,9 +171,9 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements int length = 0; int curIndex = -1; LongColumnStatsDataInspector aggregateData = null; - for (ColumnStatistics cs : css) { - String partName = cs.getStatsDesc().getPartName(); - ColumnStatisticsObj cso = cs.getStatsObjIterator().next(); + for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { + ColumnStatisticsObj cso = csp.getColStatsObj(); + String partName = csp.getPartName(); LongColumnStatsDataInspector newData = (LongColumnStatsDataInspector) cso.getStatsData().getLongStats(); // newData.isSetBitVectors() should be true for sure because we @@ -227,11 +225,13 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements } } } - extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap, - adjustedStatsMap, densityAvgSum / adjustedStatsMap.size()); + extrapolate(columnStatisticsData, partNames.size(), colStatsWithSourceInfo.size(), + adjustedIndexMap, adjustedStatsMap, densityAvgSum / adjustedStatsMap.size()); } - LOG.debug("Ndv estimatation for {} is {} # of partitions requested: {} # of partitions found: {}", colName, - columnStatisticsData.getLongStats().getNumDVs(),partNames.size(), css.size()); + LOG.debug( + "Ndv estimatation for {} is {} # of partitions requested: {} # of partitions found: {}", + colName, columnStatisticsData.getLongStats().getNumDVs(), partNames.size(), + colStatsWithSourceInfo.size()); statsObj.setStatsData(columnStatisticsData); return statsObj; } http://git-wip-us.apache.org/repos/asf/hive/blob/456a6518/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java index 4539e6b..2b8c493 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,28 +44,24 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement private static final Logger LOG = LoggerFactory.getLogger(LongColumnStatsAggregator.class); @Override - public ColumnStatisticsObj aggregate(String colName, List<String> partNames, - List<ColumnStatistics> css) throws MetaException { + public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo, + List<String> partNames, boolean areAllPartsFound) throws MetaException { ColumnStatisticsObj statsObj = null; - + String colType = null; + String colName = null; // check if all the ColumnStatisticsObjs contain stats and all the ndv are - // bitvectors. Only when both of the conditions are true, we merge bit - // vectors. Otherwise, just use the maximum function. - boolean doAllPartitionContainStats = partNames.size() == css.size(); - LOG.debug("doAllPartitionContainStats for " + colName + " is " + doAllPartitionContainStats); + // bitvectors + boolean doAllPartitionContainStats = partNames.size() == colStatsWithSourceInfo.size(); NumDistinctValueEstimator ndvEstimator = null; - String colType = null; - for (ColumnStatistics cs : css) { - if (cs.getStatsObjSize() != 1) { - throw new MetaException( - "The number of columns should be exactly one in aggrStats, but found " - + cs.getStatsObjSize()); - } - ColumnStatisticsObj cso = cs.getStatsObjIterator().next(); + for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { + ColumnStatisticsObj cso = csp.getColStatsObj(); if (statsObj == null) { + colName = cso.getColName(); colType = cso.getColType(); - statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso - .getStatsData().getSetField()); + statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, + cso.getStatsData().getSetField()); + LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, + doAllPartitionContainStats); } StringColumnStatsDataInspector stringColumnStatsData = (StringColumnStatsDataInspector) cso.getStatsData().getStringStats(); @@ -92,10 +89,10 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement } LOG.debug("all of the bit vectors can merge for " + colName + " is " + (ndvEstimator != null)); ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData(); - if (doAllPartitionContainStats || css.size() < 2) { + if (doAllPartitionContainStats || colStatsWithSourceInfo.size() < 2) { StringColumnStatsDataInspector aggregateData = null; - for (ColumnStatistics cs : css) { - ColumnStatisticsObj cso = cs.getStatsObjIterator().next(); + for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { + ColumnStatisticsObj cso = csp.getColStatsObj(); StringColumnStatsDataInspector newData = (StringColumnStatsDataInspector) cso.getStatsData().getStringStats(); if (ndvEstimator != null) { @@ -134,9 +131,9 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement if (ndvEstimator == null) { // if not every partition uses bitvector for ndv, we just fall back to // the traditional extrapolation methods. - for (ColumnStatistics cs : css) { - String partName = cs.getStatsDesc().getPartName(); - ColumnStatisticsObj cso = cs.getStatsObjIterator().next(); + for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { + ColumnStatisticsObj cso = csp.getColStatsObj(); + String partName = csp.getPartName(); adjustedIndexMap.put(partName, (double) indexMap.get(partName)); adjustedStatsMap.put(partName, cso.getStatsData()); } @@ -148,9 +145,9 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement int length = 0; int curIndex = -1; StringColumnStatsDataInspector aggregateData = null; - for (ColumnStatistics cs : css) { - String partName = cs.getStatsDesc().getPartName(); - ColumnStatisticsObj cso = cs.getStatsObjIterator().next(); + for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { + ColumnStatisticsObj cso = csp.getColStatsObj(); + String partName = csp.getPartName(); StringColumnStatsDataInspector newData = (StringColumnStatsDataInspector) cso.getStatsData().getStringStats(); // newData.isSetBitVectors() should be true for sure because we @@ -198,11 +195,13 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement adjustedStatsMap.put(pseudoPartName.toString(), csd); } } - extrapolate(columnStatisticsData, partNames.size(), css.size(), adjustedIndexMap, - adjustedStatsMap, -1); + extrapolate(columnStatisticsData, partNames.size(), colStatsWithSourceInfo.size(), + adjustedIndexMap, adjustedStatsMap, -1); } - LOG.debug("Ndv estimatation for {} is {} # of partitions requested: {} # of partitions found: {}", colName, - columnStatisticsData.getStringStats().getNumDVs(),partNames.size(), css.size()); + LOG.debug( + "Ndv estimatation for {} is {} # of partitions requested: {} # of partitions found: {}", + colName, columnStatisticsData.getStringStats().getNumDVs(), partNames.size(), + colStatsWithSourceInfo.size()); statsObj.setStatsData(columnStatisticsData); return statsObj; } http://git-wip-us.apache.org/repos/asf/hive/blob/456a6518/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java index 8bc4ce7..50f873a 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreUtils.java @@ -87,6 +87,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Map.Entry; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; @@ -213,71 +214,91 @@ public class MetaStoreUtils { } - // given a list of partStats, this function will give you an aggr stats + // Given a list of partStats, this function will give you an aggr stats public static List<ColumnStatisticsObj> aggrPartitionStats(List<ColumnStatistics> partStats, - String dbName, String tableName, List<String> partNames, List<String> colNames, - boolean useDensityFunctionForNDVEstimation, double ndvTuner) + String dbName, String tableName, List<String> partNames, List<String> colNames, + boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException { - // 1. group by the stats by colNames - // map the colName to List<ColumnStatistics> - Map<String, List<ColumnStatistics>> map = new HashMap<>(); + Map<ColumnStatsAggregator, List<ColStatsObjWithSourceInfo>> colStatsMap = + new HashMap<ColumnStatsAggregator, List<ColStatsObjWithSourceInfo>>(); + // Group stats by colName for each partition + Map<String, ColumnStatsAggregator> aliasToAggregator = + new HashMap<String, ColumnStatsAggregator>(); for (ColumnStatistics css : partStats) { List<ColumnStatisticsObj> objs = css.getStatsObj(); for (ColumnStatisticsObj obj : objs) { - List<ColumnStatisticsObj> singleObj = new ArrayList<>(); - singleObj.add(obj); - ColumnStatistics singleCS = new ColumnStatistics(css.getStatsDesc(), singleObj); - if (!map.containsKey(obj.getColName())) { - map.put(obj.getColName(), new ArrayList<>()); + String partName = css.getStatsDesc().getPartName(); + if (aliasToAggregator.get(obj.getColName()) == null) { + aliasToAggregator.put(obj.getColName(), + ColumnStatsAggregatorFactory.getColumnStatsAggregator( + obj.getStatsData().getSetField(), useDensityFunctionForNDVEstimation, ndvTuner)); + colStatsMap.put(aliasToAggregator.get(obj.getColName()), + new ArrayList<ColStatsObjWithSourceInfo>()); } - map.get(obj.getColName()).add(singleCS); + colStatsMap.get(aliasToAggregator.get(obj.getColName())) + .add(new ColStatsObjWithSourceInfo(obj, dbName, tableName, partName)); } } - return aggrPartitionStats(map,dbName,tableName,partNames,colNames,useDensityFunctionForNDVEstimation, ndvTuner); + if (colStatsMap.size() < 1) { + LOG.debug("No stats data found for: dbName= {}, tblName= {}, partNames= {}, colNames= {}", + dbName, tableName, partNames, colNames); + return new ArrayList<ColumnStatisticsObj>(); + } + return aggrPartitionStats(colStatsMap, partNames, areAllPartsFound, + useDensityFunctionForNDVEstimation, ndvTuner); } public static List<ColumnStatisticsObj> aggrPartitionStats( - Map<String, List<ColumnStatistics>> map, String dbName, String tableName, - final List<String> partNames, List<String> colNames, - final boolean useDensityFunctionForNDVEstimation,final double ndvTuner) throws MetaException { - List<ColumnStatisticsObj> colStats = new ArrayList<>(); - // 2. Aggregate stats for each column in a separate thread - if (map.size()< 1) { - //stats are absent in RDBMS - LOG.debug("No stats data found for: dbName=" +dbName +" tblName=" + tableName + - " partNames= " + partNames + " colNames=" + colNames ); - return colStats; - } - final ExecutorService pool = Executors.newFixedThreadPool(Math.min(map.size(), 16), - new ThreadFactoryBuilder().setDaemon(true).setNameFormat("aggr-col-stats-%d").build()); + Map<ColumnStatsAggregator, List<ColStatsObjWithSourceInfo>> colStatsMap, + final List<String> partNames, final boolean areAllPartsFound, + final boolean useDensityFunctionForNDVEstimation, final double ndvTuner) + throws MetaException { + List<ColumnStatisticsObj> aggrColStatObjs = new ArrayList<ColumnStatisticsObj>(); + int numProcessors = Runtime.getRuntime().availableProcessors(); + final ExecutorService pool = + Executors.newFixedThreadPool(Math.min(colStatsMap.size(), numProcessors), + new ThreadFactoryBuilder().setDaemon(true).setNameFormat("aggr-col-stats-%d").build()); final List<Future<ColumnStatisticsObj>> futures = Lists.newLinkedList(); - + LOG.debug("Aggregating column stats. Threads used: {}", + Math.min(colStatsMap.size(), numProcessors)); long start = System.currentTimeMillis(); - for (final Map.Entry<String, List<ColumnStatistics>> entry : map.entrySet()) { + for (final Entry<ColumnStatsAggregator, List<ColStatsObjWithSourceInfo>> entry : colStatsMap + .entrySet()) { futures.add(pool.submit(new Callable<ColumnStatisticsObj>() { @Override - public ColumnStatisticsObj call() throws Exception { - List<ColumnStatistics> css = entry.getValue(); - ColumnStatsAggregator aggregator = ColumnStatsAggregatorFactory.getColumnStatsAggregator(css - .iterator().next().getStatsObj().iterator().next().getStatsData().getSetField(), - useDensityFunctionForNDVEstimation, ndvTuner); - ColumnStatisticsObj statsObj = aggregator.aggregate(entry.getKey(), partNames, css); - return statsObj; - }})); + public ColumnStatisticsObj call() throws MetaException { + List<ColStatsObjWithSourceInfo> colStatWithSourceInfo = entry.getValue(); + ColumnStatsAggregator aggregator = entry.getKey(); + try { + ColumnStatisticsObj statsObj = + aggregator.aggregate(colStatWithSourceInfo, partNames, areAllPartsFound); + return statsObj; + } catch (MetaException e) { + LOG.debug(e.getMessage()); + throw e; + } + } + })); } pool.shutdown(); - for (Future<ColumnStatisticsObj> future : futures) { - try { - colStats.add(future.get()); - } catch (InterruptedException | ExecutionException e) { - pool.shutdownNow(); - LOG.debug(e.toString()); - throw new MetaException(e.toString()); + if (!futures.isEmpty()) { + for (Future<ColumnStatisticsObj> future : futures) { + try { + if (future.get() != null) { + aggrColStatObjs.add(future.get()); + } + } catch (InterruptedException | ExecutionException e) { + LOG.debug(e.getMessage()); + pool.shutdownNow(); + throw new MetaException(e.toString()); + } + } } LOG.debug("Time for aggr col stats in seconds: {} Threads used: {}", - ((System.currentTimeMillis() - (double)start))/1000, Math.min(map.size(), 16)); - return colStats; + ((System.currentTimeMillis() - (double) start)) / 1000, + Math.min(colStatsMap.size(), numProcessors)); + return aggrColStatObjs; } public static double decimalToDouble(Decimal decimal) { @@ -1562,7 +1583,6 @@ public class MetaStoreUtils { return cols; } - public static boolean isValidSchedulingPolicy(String str) { try { parseSchedulingPolicy(str); @@ -1578,4 +1598,36 @@ public class MetaStoreUtils { if ("DEFAULT".equals(schedulingPolicy)) return WMPoolSchedulingPolicy.FAIR; return Enum.valueOf(WMPoolSchedulingPolicy.class, schedulingPolicy); } + + // ColumnStatisticsObj with info about its db, table, partition (if table is partitioned) + public static class ColStatsObjWithSourceInfo { + private final ColumnStatisticsObj colStatsObj; + private final String dbName; + private final String tblName; + private final String partName; + + public ColStatsObjWithSourceInfo(ColumnStatisticsObj colStatsObj, String dbName, String tblName, + String partName) { + this.colStatsObj = colStatsObj; + this.dbName = dbName; + this.tblName = tblName; + this.partName = partName; + } + + public ColumnStatisticsObj getColStatsObj() { + return colStatsObj; + } + + public String getDbName() { + return dbName; + } + + public String getTblName() { + return tblName; + } + + public String getPartName() { + return partName; + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/456a6518/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index c43062d..877899c 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -31,7 +31,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -73,6 +72,7 @@ import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.api.WMMapping; import org.apache.hadoop.hive.metastore.api.WMPool; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; import org.apache.thrift.TException; /** @@ -937,13 +937,6 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override - public Map<String, List<ColumnStatisticsObj>> getColStatsForTablePartitions(String dbName, - String tableName) throws MetaException, NoSuchObjectException { - // TODO Auto-generated method stub - return null; - } - - @Override public String getMetastoreDbUuid() throws MetaException { throw new MetaException("Get metastore uuid is not implemented"); } @@ -1057,4 +1050,11 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { objectStore.dropWMTriggerToPoolMapping(resourcePlanName, triggerName, poolPath); } + + @Override + public List<ColStatsObjWithSourceInfo> getPartitionColStatsForDatabase(String dbName) + throws MetaException, NoSuchObjectException { + // TODO Auto-generated method stub + return null; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/456a6518/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index 688a7f9..6aa5d95 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -29,7 +29,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -72,6 +71,7 @@ import org.apache.hadoop.hive.metastore.api.WMMapping; import org.apache.hadoop.hive.metastore.api.WMPool; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; import org.apache.thrift.TException; import org.junit.Assert; @@ -949,13 +949,6 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override - public Map<String, List<ColumnStatisticsObj>> getColStatsForTablePartitions(String dbName, - String tableName) throws MetaException, NoSuchObjectException { - // TODO Auto-generated method stub - return null; - } - - @Override public String getMetastoreDbUuid() throws MetaException { throw new MetaException("Get metastore uuid is not implemented"); } @@ -1054,4 +1047,11 @@ public class DummyRawStoreForJdoConnection implements RawStore { public void dropWMTriggerToPoolMapping(String resourcePlanName, String triggerName, String poolPath) throws NoSuchObjectException, InvalidOperationException, MetaException { } + + @Override + public List<ColStatsObjWithSourceInfo> getPartitionColStatsForDatabase(String dbName) + throws MetaException, NoSuchObjectException { + // TODO Auto-generated method stub + return null; + } }
