Repository: hive Updated Branches: refs/heads/branch-3 455793cd6 -> 2f02f199e
HIVE-19316: StatsTask fails due to ClassCastException (Jaume Marhuenda, reviewed by Jesus Camacho Rodriguez) Closes apache/hive#378 Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2f02f199 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2f02f199 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2f02f199 Branch: refs/heads/branch-3 Commit: 2f02f199ed08fb93c8066e182d507672fadeb431 Parents: 455793c Author: Jaume Marhuenda <[email protected]> Authored: Mon Aug 13 23:11:42 2018 -0700 Committer: Jesus Camacho Rodriguez <[email protected]> Committed: Mon Aug 13 23:13:28 2018 -0700 ---------------------------------------------------------------------- .../columnstats/ColumnsStatsUtils.java | 117 +++++++++++++++++++ .../aggr/DateColumnStatsAggregator.java | 13 +-- .../aggr/DecimalColumnStatsAggregator.java | 12 +- .../aggr/DoubleColumnStatsAggregator.java | 9 +- .../aggr/LongColumnStatsAggregator.java | 11 +- .../aggr/StringColumnStatsAggregator.java | 13 ++- .../cache/DateColumnStatsDataInspector.java | 4 + .../cache/DecimalColumnStatsDataInspector.java | 4 + .../cache/DoubleColumnStatsDataInspector.java | 4 + .../cache/LongColumnStatsDataInspector.java | 4 + .../cache/StringColumnStatsDataInspector.java | 4 + .../merge/DateColumnStatsMerger.java | 8 +- .../merge/DecimalColumnStatsMerger.java | 6 +- .../merge/DoubleColumnStatsMerger.java | 8 +- .../merge/LongColumnStatsMerger.java | 8 +- .../merge/StringColumnStatsMerger.java | 8 +- 16 files changed, 186 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/ColumnsStatsUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/ColumnsStatsUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/ColumnsStatsUtils.java new file mode 100644 index 0000000..2d6d226 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/columnstats/ColumnsStatsUtils.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.columnstats; + +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; +import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; + +/** + * Utils class for columnstats package. + */ +public final class ColumnsStatsUtils { + + private ColumnsStatsUtils(){} + + /** + * Convertes to DateColumnStatsDataInspector if it's a DateColumnStatsData. + * @param cso ColumnStatisticsObj + * @return DateColumnStatsDataInspector + */ + public static DateColumnStatsDataInspector dateInspectorFromStats(ColumnStatisticsObj cso) { + DateColumnStatsDataInspector dateColumnStats; + if (cso.getStatsData().getDateStats() instanceof DateColumnStatsDataInspector) { + dateColumnStats = + (DateColumnStatsDataInspector)(cso.getStatsData().getDateStats()); + } else { + dateColumnStats = new DateColumnStatsDataInspector(cso.getStatsData().getDateStats()); + } + return dateColumnStats; + } + + /** + * Convertes to StringColumnStatsDataInspector + * if it's a StringColumnStatsData. + * @param cso ColumnStatisticsObj + * @return StringColumnStatsDataInspector + */ + public static StringColumnStatsDataInspector stringInspectorFromStats(ColumnStatisticsObj cso) { + StringColumnStatsDataInspector columnStats; + if (cso.getStatsData().getStringStats() instanceof StringColumnStatsDataInspector) { + columnStats = + (StringColumnStatsDataInspector)(cso.getStatsData().getStringStats()); + } else { + columnStats = new StringColumnStatsDataInspector(cso.getStatsData().getStringStats()); + } + return columnStats; + } + + /** + * Convertes to LongColumnStatsDataInspector if it's a LongColumnStatsData. + * @param cso ColumnStatisticsObj + * @return LongColumnStatsDataInspector + */ + public static LongColumnStatsDataInspector longInspectorFromStats(ColumnStatisticsObj cso) { + LongColumnStatsDataInspector columnStats; + if (cso.getStatsData().getLongStats() instanceof LongColumnStatsDataInspector) { + columnStats = + (LongColumnStatsDataInspector)(cso.getStatsData().getLongStats()); + } else { + columnStats = new LongColumnStatsDataInspector(cso.getStatsData().getLongStats()); + } + return columnStats; + } + + /** + * Convertes to DoubleColumnStatsDataInspector + * if it's a DoubleColumnStatsData. + * @param cso ColumnStatisticsObj + * @return DoubleColumnStatsDataInspector + */ + public static DoubleColumnStatsDataInspector doubleInspectorFromStats(ColumnStatisticsObj cso) { + DoubleColumnStatsDataInspector columnStats; + if (cso.getStatsData().getDoubleStats() instanceof DoubleColumnStatsDataInspector) { + columnStats = + (DoubleColumnStatsDataInspector)(cso.getStatsData().getDoubleStats()); + } else { + columnStats = new DoubleColumnStatsDataInspector(cso.getStatsData().getDoubleStats()); + } + return columnStats; + } + + /** + * Convertes to DecimalColumnStatsDataInspector + * if it's a DecimalColumnStatsData. + * @param cso ColumnStatisticsObj + * @return DecimalColumnStatsDataInspector + */ + public static DecimalColumnStatsDataInspector decimalInspectorFromStats(ColumnStatisticsObj cso) { + DecimalColumnStatsDataInspector columnStats; + if (cso.getStatsData().getDecimalStats() instanceof DecimalColumnStatsDataInspector) { + columnStats = + (DecimalColumnStatsDataInspector)(cso.getStatsData().getDecimalStats()); + } else { + columnStats = new DecimalColumnStatsDataInspector(cso.getStatsData().getDecimalStats()); + } + return columnStats; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java index e8ff513..9495424 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DateColumnStatsAggregator.java @@ -38,6 +38,8 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSour import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.dateInspectorFromStats; + public class DateColumnStatsAggregator extends ColumnStatsAggregator implements IExtrapolatePartStatus { @@ -62,8 +64,8 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements cso.getStatsData().getSetField()); LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, doAllPartitionContainStats); } - DateColumnStatsDataInspector dateColumnStats = - (DateColumnStatsDataInspector) cso.getStatsData().getDateStats(); + DateColumnStatsDataInspector dateColumnStats = dateInspectorFromStats(cso); + if (dateColumnStats.getNdvEstimator() == null) { ndvEstimator = null; break; @@ -95,9 +97,7 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements double densityAvgSum = 0.0; for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); - DateColumnStatsDataInspector newData = - (DateColumnStatsDataInspector) cso.getStatsData().getDateStats(); - lowerBound = Math.max(lowerBound, newData.getNumDVs()); + DateColumnStatsDataInspector newData = dateInspectorFromStats(cso); higherBound += newData.getNumDVs(); densityAvgSum += (diff(newData.getHighValue(), newData.getLowValue())) / newData.getNumDVs(); @@ -174,8 +174,7 @@ public class DateColumnStatsAggregator extends ColumnStatsAggregator implements for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); String partName = csp.getPartName(); - DateColumnStatsDataInspector newData = - (DateColumnStatsDataInspector) cso.getStatsData().getDateStats(); + DateColumnStatsDataInspector newData = dateInspectorFromStats(cso); // newData.isSetBitVectors() should be true for sure because we // already checked it before. if (indexMap.get(partName) != curIndex) { http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java index ac7e8e3..8739e73 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DecimalColumnStatsAggregator.java @@ -40,6 +40,8 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSour import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.decimalInspectorFromStats; + public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implements IExtrapolatePartStatus { @@ -65,8 +67,8 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, doAllPartitionContainStats); } - DecimalColumnStatsDataInspector decimalColumnStatsData = - (DecimalColumnStatsDataInspector) cso.getStatsData().getDecimalStats(); + DecimalColumnStatsDataInspector decimalColumnStatsData = decimalInspectorFromStats(cso); + if (decimalColumnStatsData.getNdvEstimator() == null) { ndvEstimator = null; break; @@ -98,8 +100,7 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen double densityAvgSum = 0.0; for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); - DecimalColumnStatsDataInspector newData = - (DecimalColumnStatsDataInspector) cso.getStatsData().getDecimalStats(); + DecimalColumnStatsDataInspector newData = decimalInspectorFromStats(cso); lowerBound = Math.max(lowerBound, newData.getNumDVs()); higherBound += newData.getNumDVs(); densityAvgSum += (MetaStoreUtils.decimalToDouble(newData.getHighValue()) - MetaStoreUtils @@ -187,8 +188,7 @@ public class DecimalColumnStatsAggregator extends ColumnStatsAggregator implemen for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); String partName = csp.getPartName(); - DecimalColumnStatsDataInspector newData = - (DecimalColumnStatsDataInspector) cso.getStatsData().getDecimalStats(); + DecimalColumnStatsDataInspector newData = decimalInspectorFromStats(cso); // newData.isSetBitVectors() should be true for sure because we // already checked it before. if (indexMap.get(partName) != curIndex) { http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java index ece77dd..5ad8453 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/DoubleColumnStatsAggregator.java @@ -37,6 +37,8 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSour import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.doubleInspectorFromStats; + public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implements IExtrapolatePartStatus { @@ -63,7 +65,7 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement doAllPartitionContainStats); } DoubleColumnStatsDataInspector doubleColumnStatsData = - (DoubleColumnStatsDataInspector) cso.getStatsData().getDoubleStats(); + doubleInspectorFromStats(cso); if (doubleColumnStatsData.getNdvEstimator() == null) { ndvEstimator = null; break; @@ -95,8 +97,7 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement double densityAvgSum = 0.0; for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); - DoubleColumnStatsDataInspector newData = - (DoubleColumnStatsDataInspector) cso.getStatsData().getDoubleStats(); + DoubleColumnStatsDataInspector newData = doubleInspectorFromStats(cso); lowerBound = Math.max(lowerBound, newData.getNumDVs()); higherBound += newData.getNumDVs(); densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs(); @@ -173,7 +174,7 @@ public class DoubleColumnStatsAggregator extends ColumnStatsAggregator implement ColumnStatisticsObj cso = csp.getColStatsObj(); String partName = csp.getPartName(); DoubleColumnStatsDataInspector newData = - (DoubleColumnStatsDataInspector) cso.getStatsData().getDoubleStats(); + doubleInspectorFromStats(cso); // newData.isSetBitVectors() should be true for sure because we // already checked it before. if (indexMap.get(partName) != curIndex) { http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java index e6823d3..ab31539 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/LongColumnStatsAggregator.java @@ -38,6 +38,8 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSour import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.longInspectorFromStats; + public class LongColumnStatsAggregator extends ColumnStatsAggregator implements IExtrapolatePartStatus { @@ -63,8 +65,7 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, doAllPartitionContainStats); } - LongColumnStatsDataInspector longColumnStatsData = - (LongColumnStatsDataInspector) cso.getStatsData().getLongStats(); + LongColumnStatsDataInspector longColumnStatsData = longInspectorFromStats(cso); if (longColumnStatsData.getNdvEstimator() == null) { ndvEstimator = null; break; @@ -96,8 +97,7 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements double densityAvgSum = 0.0; for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); - LongColumnStatsDataInspector newData = - (LongColumnStatsDataInspector) cso.getStatsData().getLongStats(); + LongColumnStatsDataInspector newData = longInspectorFromStats(cso); lowerBound = Math.max(lowerBound, newData.getNumDVs()); higherBound += newData.getNumDVs(); densityAvgSum += (newData.getHighValue() - newData.getLowValue()) / newData.getNumDVs(); @@ -174,8 +174,7 @@ public class LongColumnStatsAggregator extends ColumnStatsAggregator implements for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); String partName = csp.getPartName(); - LongColumnStatsDataInspector newData = - (LongColumnStatsDataInspector) cso.getStatsData().getLongStats(); + LongColumnStatsDataInspector newData = longInspectorFromStats(cso); // newData.isSetBitVectors() should be true for sure because we // already checked it before. if (indexMap.get(partName) != curIndex) { http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java index 9537647..92fdda5 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/aggr/StringColumnStatsAggregator.java @@ -38,6 +38,8 @@ import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSour import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.stringInspectorFromStats; + public class StringColumnStatsAggregator extends ColumnStatsAggregator implements IExtrapolatePartStatus { @@ -63,8 +65,7 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement LOG.trace("doAllPartitionContainStats for column: {} is: {}", colName, doAllPartitionContainStats); } - StringColumnStatsDataInspector stringColumnStatsData = - (StringColumnStatsDataInspector) cso.getStatsData().getStringStats(); + StringColumnStatsDataInspector stringColumnStatsData = stringInspectorFromStats(cso); if (stringColumnStatsData.getNdvEstimator() == null) { ndvEstimator = null; break; @@ -93,8 +94,7 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement StringColumnStatsDataInspector aggregateData = null; for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); - StringColumnStatsDataInspector newData = - (StringColumnStatsDataInspector) cso.getStatsData().getStringStats(); + StringColumnStatsDataInspector newData = stringInspectorFromStats(cso); if (ndvEstimator != null) { ndvEstimator.mergeEstimators(newData.getNdvEstimator()); } @@ -149,7 +149,7 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement ColumnStatisticsObj cso = csp.getColStatsObj(); String partName = csp.getPartName(); StringColumnStatsDataInspector newData = - (StringColumnStatsDataInspector) cso.getStatsData().getStringStats(); + stringInspectorFromStats(cso); // newData.isSetBitVectors() should be true for sure because we // already checked it before. if (indexMap.get(partName) != curIndex) { @@ -211,7 +211,8 @@ public class StringColumnStatsAggregator extends ColumnStatsAggregator implement int numPartsWithStats, Map<String, Double> adjustedIndexMap, Map<String, ColumnStatisticsData> adjustedStatsMap, double densityAvg) { int rightBorderInd = numParts; - StringColumnStatsDataInspector extrapolateStringData = new StringColumnStatsDataInspector(); + StringColumnStatsDataInspector extrapolateStringData = + new StringColumnStatsDataInspector(); Map<String, StringColumnStatsData> extractedAdjustedStatsMap = new HashMap<>(); for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) { extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getStringStats()); http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DateColumnStatsDataInspector.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DateColumnStatsDataInspector.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DateColumnStatsDataInspector.java index f6eacbc..d66e19a 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DateColumnStatsDataInspector.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DateColumnStatsDataInspector.java @@ -43,6 +43,10 @@ public class DateColumnStatsDataInspector extends DateColumnStatsData { } } + public DateColumnStatsDataInspector(DateColumnStatsData other) { + super(other); + } + @Override public DateColumnStatsDataInspector deepCopy() { return new DateColumnStatsDataInspector(this); http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DecimalColumnStatsDataInspector.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DecimalColumnStatsDataInspector.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DecimalColumnStatsDataInspector.java index e2427f3..88cab2c 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DecimalColumnStatsDataInspector.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DecimalColumnStatsDataInspector.java @@ -43,6 +43,10 @@ public class DecimalColumnStatsDataInspector extends DecimalColumnStatsData { } } + public DecimalColumnStatsDataInspector(DecimalColumnStatsData other) { + super(other); + } + @Override public DecimalColumnStatsDataInspector deepCopy() { return new DecimalColumnStatsDataInspector(this); http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DoubleColumnStatsDataInspector.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DoubleColumnStatsDataInspector.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DoubleColumnStatsDataInspector.java index 7ce7127..2ee7fad 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DoubleColumnStatsDataInspector.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/DoubleColumnStatsDataInspector.java @@ -43,6 +43,10 @@ public class DoubleColumnStatsDataInspector extends DoubleColumnStatsData { } } + public DoubleColumnStatsDataInspector(DoubleColumnStatsData other) { + super(other); + } + @Override public DoubleColumnStatsDataInspector deepCopy() { return new DoubleColumnStatsDataInspector(this); http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/LongColumnStatsDataInspector.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/LongColumnStatsDataInspector.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/LongColumnStatsDataInspector.java index faf314b..a4d0a1d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/LongColumnStatsDataInspector.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/LongColumnStatsDataInspector.java @@ -43,6 +43,10 @@ public class LongColumnStatsDataInspector extends LongColumnStatsData { } } + public LongColumnStatsDataInspector(LongColumnStatsData other) { + super(other); + } + @Override public LongColumnStatsDataInspector deepCopy() { return new LongColumnStatsDataInspector(this); http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java index 0876410..12afb9c 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java @@ -44,6 +44,10 @@ public class StringColumnStatsDataInspector extends StringColumnStatsData { } } + public StringColumnStatsDataInspector(StringColumnStatsData other) { + super(other); + } + @Override public StringColumnStatsDataInspector deepCopy() { return new StringColumnStatsDataInspector(this); http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java index 5baebbb..3e0ce07 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java @@ -24,13 +24,13 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Date; import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector; +import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.dateInspectorFromStats; + public class DateColumnStatsMerger extends ColumnStatsMerger { @Override public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - DateColumnStatsDataInspector aggregateData = - (DateColumnStatsDataInspector) aggregateColStats.getStatsData().getDateStats(); - DateColumnStatsDataInspector newData = - (DateColumnStatsDataInspector) newColStats.getStatsData().getDateStats(); + DateColumnStatsDataInspector aggregateData = dateInspectorFromStats(aggregateColStats); + DateColumnStatsDataInspector newData = dateInspectorFromStats(newColStats); Date lowValue = aggregateData.getLowValue().compareTo(newData.getLowValue()) < 0 ? aggregateData .getLowValue() : newData.getLowValue(); aggregateData.setLowValue(lowValue); http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java index 517ca72..5094358 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java @@ -24,13 +24,15 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Decimal; import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; +import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.decimalInspectorFromStats; + public class DecimalColumnStatsMerger extends ColumnStatsMerger { @Override public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { DecimalColumnStatsDataInspector aggregateData = - (DecimalColumnStatsDataInspector) aggregateColStats.getStatsData().getDecimalStats(); + decimalInspectorFromStats(aggregateColStats); DecimalColumnStatsDataInspector newData = - (DecimalColumnStatsDataInspector) newColStats.getStatsData().getDecimalStats(); + decimalInspectorFromStats(newColStats); Decimal lowValue = getMin(aggregateData.getLowValue(), newData.getLowValue()); aggregateData.setLowValue(lowValue); http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java index 6a95751..cbacacd 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java @@ -23,13 +23,13 @@ import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; +import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.doubleInspectorFromStats; + public class DoubleColumnStatsMerger extends ColumnStatsMerger { @Override public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - DoubleColumnStatsDataInspector aggregateData = - (DoubleColumnStatsDataInspector) aggregateColStats.getStatsData().getDoubleStats(); - DoubleColumnStatsDataInspector newData = - (DoubleColumnStatsDataInspector) newColStats.getStatsData().getDoubleStats(); + DoubleColumnStatsDataInspector aggregateData = doubleInspectorFromStats(aggregateColStats); + DoubleColumnStatsDataInspector newData = doubleInspectorFromStats(newColStats); aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue())); aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue())); aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java index ca1a912..8e70371 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java @@ -23,13 +23,13 @@ import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; +import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.longInspectorFromStats; + public class LongColumnStatsMerger extends ColumnStatsMerger { @Override public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - LongColumnStatsDataInspector aggregateData = - (LongColumnStatsDataInspector) aggregateColStats.getStatsData().getLongStats(); - LongColumnStatsDataInspector newData = - (LongColumnStatsDataInspector) newColStats.getStatsData().getLongStats(); + LongColumnStatsDataInspector aggregateData = longInspectorFromStats(aggregateColStats); + LongColumnStatsDataInspector newData = longInspectorFromStats(newColStats); aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue())); aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue())); aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); http://git-wip-us.apache.org/repos/asf/hive/blob/2f02f199/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java index d6b4478..762685d 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java @@ -23,13 +23,13 @@ import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; +import static org.apache.hadoop.hive.metastore.columnstats.ColumnsStatsUtils.stringInspectorFromStats; + public class StringColumnStatsMerger extends ColumnStatsMerger { @Override public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - StringColumnStatsDataInspector aggregateData = - (StringColumnStatsDataInspector) aggregateColStats.getStatsData().getStringStats(); - StringColumnStatsDataInspector newData = - (StringColumnStatsDataInspector) newColStats.getStatsData().getStringStats(); + StringColumnStatsDataInspector aggregateData = stringInspectorFromStats(aggregateColStats); + StringColumnStatsDataInspector newData = stringInspectorFromStats(newColStats); aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen())); aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen())); aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls());
