http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java deleted file mode 100644 index 7a0b21b..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/StatObjectConverter.java +++ /dev/null @@ -1,892 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.ByteBuffer; - -import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; -import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.api.Date; -import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; -import org.apache.hadoop.hive.metastore.api.Decimal; -import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; -import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; -import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; -import org.apache.hadoop.hive.metastore.api.utils.DecimalUtils; -import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.model.MPartition; -import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics; -import org.apache.hadoop.hive.metastore.model.MTable; -import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics; - -import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; - -/** - * This class contains conversion logic that creates Thrift stat objects from - * JDO stat objects and plain arrays from DirectSQL. - * It is hidden here so that we wouldn't have to look at it in elsewhere. - */ -public class StatObjectConverter { - // JDO - public static MTableColumnStatistics convertToMTableColumnStatistics(MTable table, - ColumnStatisticsDesc statsDesc, ColumnStatisticsObj statsObj) - throws NoSuchObjectException, MetaException, InvalidObjectException { - if (statsObj == null || statsDesc == null) { - throw new InvalidObjectException("Invalid column stats object"); - } - - MTableColumnStatistics mColStats = new MTableColumnStatistics(); - mColStats.setTable(table); - mColStats.setDbName(statsDesc.getDbName()); - mColStats.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName() : DEFAULT_CATALOG_NAME); - mColStats.setTableName(statsDesc.getTableName()); - mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed()); - mColStats.setColName(statsObj.getColName()); - mColStats.setColType(statsObj.getColType()); - - if (statsObj.getStatsData().isSetBooleanStats()) { - BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats(); - mColStats.setBooleanStats( - boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null, - boolStats.isSetNumFalses() ? boolStats.getNumFalses() : null, - boolStats.isSetNumNulls() ? boolStats.getNumNulls() : null); - } else if (statsObj.getStatsData().isSetLongStats()) { - LongColumnStatsData longStats = statsObj.getStatsData().getLongStats(); - mColStats.setLongStats( - longStats.isSetNumNulls() ? longStats.getNumNulls() : null, - longStats.isSetNumDVs() ? longStats.getNumDVs() : null, - longStats.isSetBitVectors() ? longStats.getBitVectors() : null, - longStats.isSetLowValue() ? longStats.getLowValue() : null, - longStats.isSetHighValue() ? longStats.getHighValue() : null); - } else if (statsObj.getStatsData().isSetDoubleStats()) { - DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats(); - mColStats.setDoubleStats( - doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null, - doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null, - doubleStats.isSetBitVectors() ? doubleStats.getBitVectors() : null, - doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null, - doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null); - } else if (statsObj.getStatsData().isSetDecimalStats()) { - DecimalColumnStatsData decimalStats = statsObj.getStatsData().getDecimalStats(); - String low = decimalStats.isSetLowValue() ? DecimalUtils.createJdoDecimalString(decimalStats.getLowValue()) : null; - String high = decimalStats.isSetHighValue() ? DecimalUtils.createJdoDecimalString(decimalStats.getHighValue()) : null; - mColStats.setDecimalStats( - decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null, - decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null, - decimalStats.isSetBitVectors() ? decimalStats.getBitVectors() : null, - low, high); - } else if (statsObj.getStatsData().isSetStringStats()) { - StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats(); - mColStats.setStringStats( - stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null, - stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null, - stringStats.isSetBitVectors() ? stringStats.getBitVectors() : null, - stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null, - stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null); - } else if (statsObj.getStatsData().isSetBinaryStats()) { - BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats(); - mColStats.setBinaryStats( - binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null, - binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null, - binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null); - } else if (statsObj.getStatsData().isSetDateStats()) { - DateColumnStatsData dateStats = statsObj.getStatsData().getDateStats(); - mColStats.setDateStats( - dateStats.isSetNumNulls() ? dateStats.getNumNulls() : null, - dateStats.isSetNumDVs() ? dateStats.getNumDVs() : null, - dateStats.isSetBitVectors() ? dateStats.getBitVectors() : null, - dateStats.isSetLowValue() ? dateStats.getLowValue().getDaysSinceEpoch() : null, - dateStats.isSetHighValue() ? dateStats.getHighValue().getDaysSinceEpoch() : null); - } - return mColStats; - } - - public static void setFieldsIntoOldStats( - MTableColumnStatistics mStatsObj, MTableColumnStatistics oldStatsObj) { - if (mStatsObj.getAvgColLen() != null) { - oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen()); - } - if (mStatsObj.getLongHighValue() != null) { - oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue()); - } - if (mStatsObj.getLongLowValue() != null) { - oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue()); - } - if (mStatsObj.getDoubleLowValue() != null) { - oldStatsObj.setDoubleLowValue(mStatsObj.getDoubleLowValue()); - } - if (mStatsObj.getDoubleHighValue() != null) { - oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue()); - } - if (mStatsObj.getDecimalLowValue() != null) { - oldStatsObj.setDecimalLowValue(mStatsObj.getDecimalLowValue()); - } - if (mStatsObj.getDecimalHighValue() != null) { - oldStatsObj.setDecimalHighValue(mStatsObj.getDecimalHighValue()); - } - if (mStatsObj.getMaxColLen() != null) { - oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen()); - } - if (mStatsObj.getNumDVs() != null) { - oldStatsObj.setNumDVs(mStatsObj.getNumDVs()); - } - if (mStatsObj.getBitVector() != null) { - oldStatsObj.setBitVector(mStatsObj.getBitVector()); - } - if (mStatsObj.getNumFalses() != null) { - oldStatsObj.setNumFalses(mStatsObj.getNumFalses()); - } - if (mStatsObj.getNumTrues() != null) { - oldStatsObj.setNumTrues(mStatsObj.getNumTrues()); - } - if (mStatsObj.getNumNulls() != null) { - oldStatsObj.setNumNulls(mStatsObj.getNumNulls()); - } - oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed()); - } - - public static void setFieldsIntoOldStats( - MPartitionColumnStatistics mStatsObj, MPartitionColumnStatistics oldStatsObj) { - if (mStatsObj.getAvgColLen() != null) { - oldStatsObj.setAvgColLen(mStatsObj.getAvgColLen()); - } - if (mStatsObj.getLongHighValue() != null) { - oldStatsObj.setLongHighValue(mStatsObj.getLongHighValue()); - } - if (mStatsObj.getDoubleHighValue() != null) { - oldStatsObj.setDoubleHighValue(mStatsObj.getDoubleHighValue()); - } - oldStatsObj.setLastAnalyzed(mStatsObj.getLastAnalyzed()); - if (mStatsObj.getLongLowValue() != null) { - oldStatsObj.setLongLowValue(mStatsObj.getLongLowValue()); - } - if (mStatsObj.getDoubleLowValue() != null) { - oldStatsObj.setDoubleLowValue(mStatsObj.getDoubleLowValue()); - } - if (mStatsObj.getDecimalLowValue() != null) { - oldStatsObj.setDecimalLowValue(mStatsObj.getDecimalLowValue()); - } - if (mStatsObj.getDecimalHighValue() != null) { - oldStatsObj.setDecimalHighValue(mStatsObj.getDecimalHighValue()); - } - if (mStatsObj.getMaxColLen() != null) { - oldStatsObj.setMaxColLen(mStatsObj.getMaxColLen()); - } - if (mStatsObj.getNumDVs() != null) { - oldStatsObj.setNumDVs(mStatsObj.getNumDVs()); - } - if (mStatsObj.getBitVector() != null) { - oldStatsObj.setBitVector(mStatsObj.getBitVector()); - } - if (mStatsObj.getNumFalses() != null) { - oldStatsObj.setNumFalses(mStatsObj.getNumFalses()); - } - if (mStatsObj.getNumTrues() != null) { - oldStatsObj.setNumTrues(mStatsObj.getNumTrues()); - } - if (mStatsObj.getNumNulls() != null) { - oldStatsObj.setNumNulls(mStatsObj.getNumNulls()); - } - } - - public static ColumnStatisticsObj getTableColumnStatisticsObj( - MTableColumnStatistics mStatsObj, boolean enableBitVector) { - ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); - statsObj.setColType(mStatsObj.getColType()); - statsObj.setColName(mStatsObj.getColName()); - String colType = mStatsObj.getColType().toLowerCase(); - ColumnStatisticsData colStatsData = new ColumnStatisticsData(); - - if (colType.equals("boolean")) { - BooleanColumnStatsData boolStats = new BooleanColumnStatsData(); - boolStats.setNumFalses(mStatsObj.getNumFalses()); - boolStats.setNumTrues(mStatsObj.getNumTrues()); - boolStats.setNumNulls(mStatsObj.getNumNulls()); - colStatsData.setBooleanStats(boolStats); - } else if (colType.equals("string") || - colType.startsWith("varchar") || colType.startsWith("char")) { - StringColumnStatsDataInspector stringStats = new StringColumnStatsDataInspector(); - stringStats.setNumNulls(mStatsObj.getNumNulls()); - stringStats.setAvgColLen(mStatsObj.getAvgColLen()); - stringStats.setMaxColLen(mStatsObj.getMaxColLen()); - stringStats.setNumDVs(mStatsObj.getNumDVs()); - stringStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); - colStatsData.setStringStats(stringStats); - } else if (colType.equals("binary")) { - BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); - binaryStats.setNumNulls(mStatsObj.getNumNulls()); - binaryStats.setAvgColLen(mStatsObj.getAvgColLen()); - binaryStats.setMaxColLen(mStatsObj.getMaxColLen()); - colStatsData.setBinaryStats(binaryStats); - } else if (colType.equals("bigint") || colType.equals("int") || - colType.equals("smallint") || colType.equals("tinyint") || - colType.equals("timestamp")) { - LongColumnStatsDataInspector longStats = new LongColumnStatsDataInspector(); - longStats.setNumNulls(mStatsObj.getNumNulls()); - Long longHighValue = mStatsObj.getLongHighValue(); - if (longHighValue != null) { - longStats.setHighValue(longHighValue); - } - Long longLowValue = mStatsObj.getLongLowValue(); - if (longLowValue != null) { - longStats.setLowValue(longLowValue); - } - longStats.setNumDVs(mStatsObj.getNumDVs()); - longStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); - colStatsData.setLongStats(longStats); - } else if (colType.equals("double") || colType.equals("float")) { - DoubleColumnStatsDataInspector doubleStats = new DoubleColumnStatsDataInspector(); - doubleStats.setNumNulls(mStatsObj.getNumNulls()); - Double doubleHighValue = mStatsObj.getDoubleHighValue(); - if (doubleHighValue != null) { - doubleStats.setHighValue(doubleHighValue); - } - Double doubleLowValue = mStatsObj.getDoubleLowValue(); - if (doubleLowValue != null) { - doubleStats.setLowValue(doubleLowValue); - } - doubleStats.setNumDVs(mStatsObj.getNumDVs()); - doubleStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); - colStatsData.setDoubleStats(doubleStats); - } else if (colType.startsWith("decimal")) { - DecimalColumnStatsDataInspector decimalStats = new DecimalColumnStatsDataInspector(); - decimalStats.setNumNulls(mStatsObj.getNumNulls()); - String decimalHighValue = mStatsObj.getDecimalHighValue(); - if (decimalHighValue != null) { - decimalStats.setHighValue(DecimalUtils.createThriftDecimal(decimalHighValue)); - } - String decimalLowValue = mStatsObj.getDecimalLowValue(); - if (decimalLowValue != null) { - decimalStats.setLowValue(DecimalUtils.createThriftDecimal(decimalLowValue)); - } - decimalStats.setNumDVs(mStatsObj.getNumDVs()); - decimalStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); - colStatsData.setDecimalStats(decimalStats); - } else if (colType.equals("date")) { - DateColumnStatsDataInspector dateStats = new DateColumnStatsDataInspector(); - dateStats.setNumNulls(mStatsObj.getNumNulls()); - Long highValue = mStatsObj.getLongHighValue(); - if (highValue != null) { - dateStats.setHighValue(new Date(highValue)); - } - Long lowValue = mStatsObj.getLongLowValue(); - if (lowValue != null) { - dateStats.setLowValue(new Date(lowValue)); - } - dateStats.setNumDVs(mStatsObj.getNumDVs()); - dateStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); - colStatsData.setDateStats(dateStats); - } - statsObj.setStatsData(colStatsData); - return statsObj; - } - - public static ColumnStatisticsDesc getTableColumnStatisticsDesc( - MTableColumnStatistics mStatsObj) { - ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); - statsDesc.setIsTblLevel(true); - statsDesc.setCatName(mStatsObj.getCatName()); - statsDesc.setDbName(mStatsObj.getDbName()); - statsDesc.setTableName(mStatsObj.getTableName()); - statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed()); - return statsDesc; - } - - public static MPartitionColumnStatistics convertToMPartitionColumnStatistics( - MPartition partition, ColumnStatisticsDesc statsDesc, ColumnStatisticsObj statsObj) - throws MetaException, NoSuchObjectException { - if (statsDesc == null || statsObj == null) { - return null; - } - - MPartitionColumnStatistics mColStats = new MPartitionColumnStatistics(); - mColStats.setPartition(partition); - mColStats.setCatName(statsDesc.isSetCatName() ? statsDesc.getCatName() : DEFAULT_CATALOG_NAME); - mColStats.setDbName(statsDesc.getDbName()); - mColStats.setTableName(statsDesc.getTableName()); - mColStats.setPartitionName(statsDesc.getPartName()); - mColStats.setLastAnalyzed(statsDesc.getLastAnalyzed()); - mColStats.setColName(statsObj.getColName()); - mColStats.setColType(statsObj.getColType()); - - if (statsObj.getStatsData().isSetBooleanStats()) { - BooleanColumnStatsData boolStats = statsObj.getStatsData().getBooleanStats(); - mColStats.setBooleanStats( - boolStats.isSetNumTrues() ? boolStats.getNumTrues() : null, - boolStats.isSetNumFalses() ? boolStats.getNumFalses() : null, - boolStats.isSetNumNulls() ? boolStats.getNumNulls() : null); - } else if (statsObj.getStatsData().isSetLongStats()) { - LongColumnStatsData longStats = statsObj.getStatsData().getLongStats(); - mColStats.setLongStats( - longStats.isSetNumNulls() ? longStats.getNumNulls() : null, - longStats.isSetNumDVs() ? longStats.getNumDVs() : null, - longStats.isSetBitVectors() ? longStats.getBitVectors() : null, - longStats.isSetLowValue() ? longStats.getLowValue() : null, - longStats.isSetHighValue() ? longStats.getHighValue() : null); - } else if (statsObj.getStatsData().isSetDoubleStats()) { - DoubleColumnStatsData doubleStats = statsObj.getStatsData().getDoubleStats(); - mColStats.setDoubleStats( - doubleStats.isSetNumNulls() ? doubleStats.getNumNulls() : null, - doubleStats.isSetNumDVs() ? doubleStats.getNumDVs() : null, - doubleStats.isSetBitVectors() ? doubleStats.getBitVectors() : null, - doubleStats.isSetLowValue() ? doubleStats.getLowValue() : null, - doubleStats.isSetHighValue() ? doubleStats.getHighValue() : null); - } else if (statsObj.getStatsData().isSetDecimalStats()) { - DecimalColumnStatsData decimalStats = statsObj.getStatsData().getDecimalStats(); - String low = decimalStats.isSetLowValue() ? DecimalUtils.createJdoDecimalString(decimalStats.getLowValue()) : null; - String high = decimalStats.isSetHighValue() ? DecimalUtils.createJdoDecimalString(decimalStats.getHighValue()) : null; - mColStats.setDecimalStats( - decimalStats.isSetNumNulls() ? decimalStats.getNumNulls() : null, - decimalStats.isSetNumDVs() ? decimalStats.getNumDVs() : null, - decimalStats.isSetBitVectors() ? decimalStats.getBitVectors() : null, - low, high); - } else if (statsObj.getStatsData().isSetStringStats()) { - StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats(); - mColStats.setStringStats( - stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null, - stringStats.isSetNumDVs() ? stringStats.getNumDVs() : null, - stringStats.isSetBitVectors() ? stringStats.getBitVectors() : null, - stringStats.isSetMaxColLen() ? stringStats.getMaxColLen() : null, - stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null); - } else if (statsObj.getStatsData().isSetBinaryStats()) { - BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats(); - mColStats.setBinaryStats( - binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null, - binaryStats.isSetMaxColLen() ? binaryStats.getMaxColLen() : null, - binaryStats.isSetAvgColLen() ? binaryStats.getAvgColLen() : null); - } else if (statsObj.getStatsData().isSetDateStats()) { - DateColumnStatsData dateStats = statsObj.getStatsData().getDateStats(); - mColStats.setDateStats( - dateStats.isSetNumNulls() ? dateStats.getNumNulls() : null, - dateStats.isSetNumDVs() ? dateStats.getNumDVs() : null, - dateStats.isSetBitVectors() ? dateStats.getBitVectors() : null, - dateStats.isSetLowValue() ? dateStats.getLowValue().getDaysSinceEpoch() : null, - dateStats.isSetHighValue() ? dateStats.getHighValue().getDaysSinceEpoch() : null); - } - return mColStats; - } - - public static ColumnStatisticsObj getPartitionColumnStatisticsObj( - MPartitionColumnStatistics mStatsObj, boolean enableBitVector) { - ColumnStatisticsObj statsObj = new ColumnStatisticsObj(); - statsObj.setColType(mStatsObj.getColType()); - statsObj.setColName(mStatsObj.getColName()); - String colType = mStatsObj.getColType().toLowerCase(); - ColumnStatisticsData colStatsData = new ColumnStatisticsData(); - - if (colType.equals("boolean")) { - BooleanColumnStatsData boolStats = new BooleanColumnStatsData(); - boolStats.setNumFalses(mStatsObj.getNumFalses()); - boolStats.setNumTrues(mStatsObj.getNumTrues()); - boolStats.setNumNulls(mStatsObj.getNumNulls()); - colStatsData.setBooleanStats(boolStats); - } else if (colType.equals("string") || - colType.startsWith("varchar") || colType.startsWith("char")) { - StringColumnStatsDataInspector stringStats = new StringColumnStatsDataInspector(); - stringStats.setNumNulls(mStatsObj.getNumNulls()); - stringStats.setAvgColLen(mStatsObj.getAvgColLen()); - stringStats.setMaxColLen(mStatsObj.getMaxColLen()); - stringStats.setNumDVs(mStatsObj.getNumDVs()); - stringStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); - colStatsData.setStringStats(stringStats); - } else if (colType.equals("binary")) { - BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); - binaryStats.setNumNulls(mStatsObj.getNumNulls()); - binaryStats.setAvgColLen(mStatsObj.getAvgColLen()); - binaryStats.setMaxColLen(mStatsObj.getMaxColLen()); - colStatsData.setBinaryStats(binaryStats); - } else if (colType.equals("tinyint") || colType.equals("smallint") || - colType.equals("int") || colType.equals("bigint") || - colType.equals("timestamp")) { - LongColumnStatsDataInspector longStats = new LongColumnStatsDataInspector(); - longStats.setNumNulls(mStatsObj.getNumNulls()); - if (mStatsObj.getLongHighValue() != null) { - longStats.setHighValue(mStatsObj.getLongHighValue()); - } - if (mStatsObj.getLongLowValue() != null) { - longStats.setLowValue(mStatsObj.getLongLowValue()); - } - longStats.setNumDVs(mStatsObj.getNumDVs()); - longStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); - colStatsData.setLongStats(longStats); - } else if (colType.equals("double") || colType.equals("float")) { - DoubleColumnStatsDataInspector doubleStats = new DoubleColumnStatsDataInspector(); - doubleStats.setNumNulls(mStatsObj.getNumNulls()); - if (mStatsObj.getDoubleHighValue() != null) { - doubleStats.setHighValue(mStatsObj.getDoubleHighValue()); - } - if (mStatsObj.getDoubleLowValue() != null) { - doubleStats.setLowValue(mStatsObj.getDoubleLowValue()); - } - doubleStats.setNumDVs(mStatsObj.getNumDVs()); - doubleStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); - colStatsData.setDoubleStats(doubleStats); - } else if (colType.startsWith("decimal")) { - DecimalColumnStatsDataInspector decimalStats = new DecimalColumnStatsDataInspector(); - decimalStats.setNumNulls(mStatsObj.getNumNulls()); - if (mStatsObj.getDecimalHighValue() != null) { - decimalStats.setHighValue(DecimalUtils.createThriftDecimal(mStatsObj.getDecimalHighValue())); - } - if (mStatsObj.getDecimalLowValue() != null) { - decimalStats.setLowValue(DecimalUtils.createThriftDecimal(mStatsObj.getDecimalLowValue())); - } - decimalStats.setNumDVs(mStatsObj.getNumDVs()); - decimalStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); - colStatsData.setDecimalStats(decimalStats); - } else if (colType.equals("date")) { - DateColumnStatsDataInspector dateStats = new DateColumnStatsDataInspector(); - dateStats.setNumNulls(mStatsObj.getNumNulls()); - Long highValue = mStatsObj.getLongHighValue(); - if (highValue != null) { - dateStats.setHighValue(new Date(highValue)); - } - Long lowValue = mStatsObj.getLongLowValue(); - if (lowValue != null) { - dateStats.setLowValue(new Date(lowValue)); - } - dateStats.setNumDVs(mStatsObj.getNumDVs()); - dateStats.setBitVectors((mStatsObj.getBitVector()==null||!enableBitVector)? null : mStatsObj.getBitVector()); - colStatsData.setDateStats(dateStats); - } - statsObj.setStatsData(colStatsData); - return statsObj; - } - - public static ColumnStatisticsDesc getPartitionColumnStatisticsDesc( - MPartitionColumnStatistics mStatsObj) { - ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(); - statsDesc.setIsTblLevel(false); - statsDesc.setCatName(mStatsObj.getCatName()); - statsDesc.setDbName(mStatsObj.getDbName()); - statsDesc.setTableName(mStatsObj.getTableName()); - statsDesc.setPartName(mStatsObj.getPartitionName()); - statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed()); - return statsDesc; - } - - // JAVA - public static void fillColumnStatisticsData(String colType, ColumnStatisticsData data, - Object llow, Object lhigh, Object dlow, Object dhigh, Object declow, Object dechigh, - Object nulls, Object dist, Object bitVector, Object avglen, Object maxlen, Object trues, Object falses) throws MetaException { - colType = colType.toLowerCase(); - if (colType.equals("boolean")) { - BooleanColumnStatsData boolStats = new BooleanColumnStatsData(); - boolStats.setNumFalses(MetaStoreDirectSql.extractSqlLong(falses)); - boolStats.setNumTrues(MetaStoreDirectSql.extractSqlLong(trues)); - boolStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - data.setBooleanStats(boolStats); - } else if (colType.equals("string") || - colType.startsWith("varchar") || colType.startsWith("char")) { - StringColumnStatsDataInspector stringStats = new StringColumnStatsDataInspector(); - stringStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - stringStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); - stringStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); - stringStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist)); - stringStats.setBitVectors(MetaStoreDirectSql.extractSqlBlob(bitVector)); - data.setStringStats(stringStats); - } else if (colType.equals("binary")) { - BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); - binaryStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - binaryStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); - binaryStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); - data.setBinaryStats(binaryStats); - } else if (colType.equals("bigint") || colType.equals("int") || - colType.equals("smallint") || colType.equals("tinyint") || - colType.equals("timestamp")) { - LongColumnStatsDataInspector longStats = new LongColumnStatsDataInspector(); - longStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - if (lhigh != null) { - longStats.setHighValue(MetaStoreDirectSql.extractSqlLong(lhigh)); - } - if (llow != null) { - longStats.setLowValue(MetaStoreDirectSql.extractSqlLong(llow)); - } - longStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist)); - longStats.setBitVectors(MetaStoreDirectSql.extractSqlBlob(bitVector)); - data.setLongStats(longStats); - } else if (colType.equals("double") || colType.equals("float")) { - DoubleColumnStatsDataInspector doubleStats = new DoubleColumnStatsDataInspector(); - doubleStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - if (dhigh != null) { - doubleStats.setHighValue(MetaStoreDirectSql.extractSqlDouble(dhigh)); - } - if (dlow != null) { - doubleStats.setLowValue(MetaStoreDirectSql.extractSqlDouble(dlow)); - } - doubleStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist)); - doubleStats.setBitVectors(MetaStoreDirectSql.extractSqlBlob(bitVector)); - data.setDoubleStats(doubleStats); - } else if (colType.startsWith("decimal")) { - DecimalColumnStatsDataInspector decimalStats = new DecimalColumnStatsDataInspector(); - decimalStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - if (dechigh != null) { - decimalStats.setHighValue(DecimalUtils.createThriftDecimal((String)dechigh)); - } - if (declow != null) { - decimalStats.setLowValue(DecimalUtils.createThriftDecimal((String)declow)); - } - decimalStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist)); - decimalStats.setBitVectors(MetaStoreDirectSql.extractSqlBlob(bitVector)); - data.setDecimalStats(decimalStats); - } else if (colType.equals("date")) { - DateColumnStatsDataInspector dateStats = new DateColumnStatsDataInspector(); - dateStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - if (lhigh != null) { - dateStats.setHighValue(new Date(MetaStoreDirectSql.extractSqlLong(lhigh))); - } - if (llow != null) { - dateStats.setLowValue(new Date(MetaStoreDirectSql.extractSqlLong(llow))); - } - dateStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist)); - dateStats.setBitVectors(MetaStoreDirectSql.extractSqlBlob(bitVector)); - data.setDateStats(dateStats); - } - } - - //DB - public static void fillColumnStatisticsData(String colType, ColumnStatisticsData data, - Object llow, Object lhigh, Object dlow, Object dhigh, Object declow, Object dechigh, - Object nulls, Object dist, Object avglen, Object maxlen, Object trues, Object falses, - Object avgLong, Object avgDouble, Object avgDecimal, Object sumDist, - boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException { - colType = colType.toLowerCase(); - if (colType.equals("boolean")) { - BooleanColumnStatsData boolStats = new BooleanColumnStatsData(); - boolStats.setNumFalses(MetaStoreDirectSql.extractSqlLong(falses)); - boolStats.setNumTrues(MetaStoreDirectSql.extractSqlLong(trues)); - boolStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - data.setBooleanStats(boolStats); - } else if (colType.equals("string") || colType.startsWith("varchar") - || colType.startsWith("char")) { - StringColumnStatsDataInspector stringStats = new StringColumnStatsDataInspector(); - stringStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - stringStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); - stringStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); - stringStats.setNumDVs(MetaStoreDirectSql.extractSqlLong(dist)); - data.setStringStats(stringStats); - } else if (colType.equals("binary")) { - BinaryColumnStatsData binaryStats = new BinaryColumnStatsData(); - binaryStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - binaryStats.setAvgColLen(MetaStoreDirectSql.extractSqlDouble(avglen)); - binaryStats.setMaxColLen(MetaStoreDirectSql.extractSqlLong(maxlen)); - data.setBinaryStats(binaryStats); - } else if (colType.equals("bigint") || colType.equals("int") || colType.equals("smallint") - || colType.equals("tinyint") || colType.equals("timestamp")) { - LongColumnStatsDataInspector longStats = new LongColumnStatsDataInspector(); - longStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - if (lhigh != null) { - longStats.setHighValue(MetaStoreDirectSql.extractSqlLong(lhigh)); - } - if (llow != null) { - longStats.setLowValue(MetaStoreDirectSql.extractSqlLong(llow)); - } - long lowerBound = MetaStoreDirectSql.extractSqlLong(dist); - long higherBound = MetaStoreDirectSql.extractSqlLong(sumDist); - long rangeBound = Long.MAX_VALUE; - if (lhigh != null && llow != null) { - rangeBound = MetaStoreDirectSql.extractSqlLong(lhigh) - - MetaStoreDirectSql.extractSqlLong(llow) + 1; - } - long estimation; - if (useDensityFunctionForNDVEstimation && lhigh != null && llow != null && avgLong != null - && MetaStoreDirectSql.extractSqlDouble(avgLong) != 0.0) { - // We have estimation, lowerbound and higherbound. We use estimation if - // it is between lowerbound and higherbound. - estimation = MetaStoreDirectSql - .extractSqlLong((MetaStoreDirectSql.extractSqlLong(lhigh) - MetaStoreDirectSql - .extractSqlLong(llow)) / MetaStoreDirectSql.extractSqlDouble(avgLong)); - if (estimation < lowerBound) { - estimation = lowerBound; - } else if (estimation > higherBound) { - estimation = higherBound; - } - } else { - estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner); - } - estimation = Math.min(estimation, rangeBound); - longStats.setNumDVs(estimation); - data.setLongStats(longStats); - } else if (colType.equals("date")) { - DateColumnStatsDataInspector dateStats = new DateColumnStatsDataInspector(); - dateStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - if (lhigh != null) { - dateStats.setHighValue(new Date(MetaStoreDirectSql.extractSqlLong(lhigh))); - } - if (llow != null) { - dateStats.setLowValue(new Date(MetaStoreDirectSql.extractSqlLong(llow))); - } - long lowerBound = MetaStoreDirectSql.extractSqlLong(dist); - long higherBound = MetaStoreDirectSql.extractSqlLong(sumDist); - long rangeBound = Long.MAX_VALUE; - if (lhigh != null && llow != null) { - rangeBound = MetaStoreDirectSql.extractSqlLong(lhigh) - - MetaStoreDirectSql.extractSqlLong(llow) + 1; - } - long estimation; - if (useDensityFunctionForNDVEstimation && lhigh != null && llow != null && avgLong != null - && MetaStoreDirectSql.extractSqlDouble(avgLong) != 0.0) { - // We have estimation, lowerbound and higherbound. We use estimation if - // it is between lowerbound and higherbound. - estimation = MetaStoreDirectSql - .extractSqlLong((MetaStoreDirectSql.extractSqlLong(lhigh) - MetaStoreDirectSql - .extractSqlLong(llow)) / MetaStoreDirectSql.extractSqlDouble(avgLong)); - if (estimation < lowerBound) { - estimation = lowerBound; - } else if (estimation > higherBound) { - estimation = higherBound; - } - } else { - estimation = (long) (lowerBound + (higherBound - lowerBound) * ndvTuner); - } - estimation = Math.min(estimation, rangeBound); - dateStats.setNumDVs(estimation); - data.setDateStats(dateStats); - } else if (colType.equals("double") || colType.equals("float")) { - DoubleColumnStatsDataInspector doubleStats = new DoubleColumnStatsDataInspector(); - doubleStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - if (dhigh != null) { - doubleStats.setHighValue(MetaStoreDirectSql.extractSqlDouble(dhigh)); - } - if (dlow != null) { - doubleStats.setLowValue(MetaStoreDirectSql.extractSqlDouble(dlow)); - } - long lowerBound = MetaStoreDirectSql.extractSqlLong(dist); - long higherBound = MetaStoreDirectSql.extractSqlLong(sumDist); - if (useDensityFunctionForNDVEstimation && dhigh != null && dlow != null && avgDouble != null - && MetaStoreDirectSql.extractSqlDouble(avgDouble) != 0.0) { - long estimation = MetaStoreDirectSql - .extractSqlLong((MetaStoreDirectSql.extractSqlLong(dhigh) - MetaStoreDirectSql - .extractSqlLong(dlow)) / MetaStoreDirectSql.extractSqlDouble(avgDouble)); - if (estimation < lowerBound) { - doubleStats.setNumDVs(lowerBound); - } else if (estimation > higherBound) { - doubleStats.setNumDVs(higherBound); - } else { - doubleStats.setNumDVs(estimation); - } - } else { - doubleStats.setNumDVs((long) (lowerBound + (higherBound - lowerBound) * ndvTuner)); - } - data.setDoubleStats(doubleStats); - } else if (colType.startsWith("decimal")) { - DecimalColumnStatsDataInspector decimalStats = new DecimalColumnStatsDataInspector(); - decimalStats.setNumNulls(MetaStoreDirectSql.extractSqlLong(nulls)); - Decimal low = null; - Decimal high = null; - BigDecimal blow = null; - BigDecimal bhigh = null; - if (dechigh instanceof BigDecimal) { - bhigh = (BigDecimal) dechigh; - high = DecimalUtils.getDecimal(ByteBuffer.wrap(bhigh.unscaledValue().toByteArray()), - (short) bhigh.scale()); - } else if (dechigh instanceof String) { - bhigh = new BigDecimal((String) dechigh); - high = DecimalUtils.createThriftDecimal((String) dechigh); - } - decimalStats.setHighValue(high); - if (declow instanceof BigDecimal) { - blow = (BigDecimal) declow; - low = DecimalUtils.getDecimal(ByteBuffer.wrap(blow.unscaledValue().toByteArray()), (short) blow.scale()); - } else if (dechigh instanceof String) { - blow = new BigDecimal((String) declow); - low = DecimalUtils.createThriftDecimal((String) declow); - } - decimalStats.setLowValue(low); - long lowerBound = MetaStoreDirectSql.extractSqlLong(dist); - long higherBound = MetaStoreDirectSql.extractSqlLong(sumDist); - if (useDensityFunctionForNDVEstimation && dechigh != null && declow != null && avgDecimal != null - && MetaStoreDirectSql.extractSqlDouble(avgDecimal) != 0.0) { - long estimation = MetaStoreDirectSql.extractSqlLong(MetaStoreDirectSql.extractSqlLong(bhigh - .subtract(blow).floatValue() / MetaStoreDirectSql.extractSqlDouble(avgDecimal))); - if (estimation < lowerBound) { - decimalStats.setNumDVs(lowerBound); - } else if (estimation > higherBound) { - decimalStats.setNumDVs(higherBound); - } else { - decimalStats.setNumDVs(estimation); - } - } else { - decimalStats.setNumDVs((long) (lowerBound + (higherBound - lowerBound) * ndvTuner)); - } - data.setDecimalStats(decimalStats); - } - } - - /** - * Set field values in oldStatObj from newStatObj - * @param oldStatObj - * @param newStatObj - */ - public static void setFieldsIntoOldStats(ColumnStatisticsObj oldStatObj, - ColumnStatisticsObj newStatObj) { - _Fields typeNew = newStatObj.getStatsData().getSetField(); - _Fields typeOld = oldStatObj.getStatsData().getSetField(); - typeNew = typeNew == typeOld ? typeNew : null; - switch (typeNew) { - case BOOLEAN_STATS: - BooleanColumnStatsData oldBooleanStatsData = oldStatObj.getStatsData().getBooleanStats(); - BooleanColumnStatsData newBooleanStatsData = newStatObj.getStatsData().getBooleanStats(); - if (newBooleanStatsData.isSetNumTrues()) { - oldBooleanStatsData.setNumTrues(newBooleanStatsData.getNumTrues()); - } - if (newBooleanStatsData.isSetNumFalses()) { - oldBooleanStatsData.setNumFalses(newBooleanStatsData.getNumFalses()); - } - if (newBooleanStatsData.isSetNumNulls()) { - oldBooleanStatsData.setNumNulls(newBooleanStatsData.getNumNulls()); - } - if (newBooleanStatsData.isSetBitVectors()) { - oldBooleanStatsData.setBitVectors(newBooleanStatsData.getBitVectors()); - } - break; - case LONG_STATS: { - LongColumnStatsData oldLongStatsData = oldStatObj.getStatsData().getLongStats(); - LongColumnStatsData newLongStatsData = newStatObj.getStatsData().getLongStats(); - if (newLongStatsData.isSetHighValue()) { - oldLongStatsData.setHighValue(newLongStatsData.getHighValue()); - } - if (newLongStatsData.isSetLowValue()) { - oldLongStatsData.setLowValue(newLongStatsData.getLowValue()); - } - if (newLongStatsData.isSetNumNulls()) { - oldLongStatsData.setNumNulls(newLongStatsData.getNumNulls()); - } - if (newLongStatsData.isSetNumDVs()) { - oldLongStatsData.setNumDVs(newLongStatsData.getNumDVs()); - } - if (newLongStatsData.isSetBitVectors()) { - oldLongStatsData.setBitVectors(newLongStatsData.getBitVectors()); - } - break; - } - case DOUBLE_STATS: { - DoubleColumnStatsData oldDoubleStatsData = oldStatObj.getStatsData().getDoubleStats(); - DoubleColumnStatsData newDoubleStatsData = newStatObj.getStatsData().getDoubleStats(); - if (newDoubleStatsData.isSetHighValue()) { - oldDoubleStatsData.setHighValue(newDoubleStatsData.getHighValue()); - } - if (newDoubleStatsData.isSetLowValue()) { - oldDoubleStatsData.setLowValue(newDoubleStatsData.getLowValue()); - } - if (newDoubleStatsData.isSetNumNulls()) { - oldDoubleStatsData.setNumNulls(newDoubleStatsData.getNumNulls()); - } - if (newDoubleStatsData.isSetNumDVs()) { - oldDoubleStatsData.setNumDVs(newDoubleStatsData.getNumDVs()); - } - if (newDoubleStatsData.isSetBitVectors()) { - oldDoubleStatsData.setBitVectors(newDoubleStatsData.getBitVectors()); - } - break; - } - case STRING_STATS: { - StringColumnStatsData oldStringStatsData = oldStatObj.getStatsData().getStringStats(); - StringColumnStatsData newStringStatsData = newStatObj.getStatsData().getStringStats(); - if (newStringStatsData.isSetMaxColLen()) { - oldStringStatsData.setMaxColLen(newStringStatsData.getMaxColLen()); - } - if (newStringStatsData.isSetAvgColLen()) { - oldStringStatsData.setAvgColLen(newStringStatsData.getAvgColLen()); - } - if (newStringStatsData.isSetNumNulls()) { - oldStringStatsData.setNumNulls(newStringStatsData.getNumNulls()); - } - if (newStringStatsData.isSetNumDVs()) { - oldStringStatsData.setNumDVs(newStringStatsData.getNumDVs()); - } - if (newStringStatsData.isSetBitVectors()) { - oldStringStatsData.setBitVectors(newStringStatsData.getBitVectors()); - } - break; - } - case BINARY_STATS: - BinaryColumnStatsData oldBinaryStatsData = oldStatObj.getStatsData().getBinaryStats(); - BinaryColumnStatsData newBinaryStatsData = newStatObj.getStatsData().getBinaryStats(); - if (newBinaryStatsData.isSetMaxColLen()) { - oldBinaryStatsData.setMaxColLen(newBinaryStatsData.getMaxColLen()); - } - if (newBinaryStatsData.isSetAvgColLen()) { - oldBinaryStatsData.setAvgColLen(newBinaryStatsData.getAvgColLen()); - } - if (newBinaryStatsData.isSetNumNulls()) { - oldBinaryStatsData.setNumNulls(newBinaryStatsData.getNumNulls()); - } - if (newBinaryStatsData.isSetBitVectors()) { - oldBinaryStatsData.setBitVectors(newBinaryStatsData.getBitVectors()); - } - break; - case DECIMAL_STATS: { - DecimalColumnStatsData oldDecimalStatsData = oldStatObj.getStatsData().getDecimalStats(); - DecimalColumnStatsData newDecimalStatsData = newStatObj.getStatsData().getDecimalStats(); - if (newDecimalStatsData.isSetHighValue()) { - oldDecimalStatsData.setHighValue(newDecimalStatsData.getHighValue()); - } - if (newDecimalStatsData.isSetLowValue()) { - oldDecimalStatsData.setLowValue(newDecimalStatsData.getLowValue()); - } - if (newDecimalStatsData.isSetNumNulls()) { - oldDecimalStatsData.setNumNulls(newDecimalStatsData.getNumNulls()); - } - if (newDecimalStatsData.isSetNumDVs()) { - oldDecimalStatsData.setNumDVs(newDecimalStatsData.getNumDVs()); - } - if (newDecimalStatsData.isSetBitVectors()) { - oldDecimalStatsData.setBitVectors(newDecimalStatsData.getBitVectors()); - } - break; - } - case DATE_STATS: { - DateColumnStatsData oldDateStatsData = oldStatObj.getStatsData().getDateStats(); - DateColumnStatsData newDateStatsData = newStatObj.getStatsData().getDateStats(); - if (newDateStatsData.isSetHighValue()) { - oldDateStatsData.setHighValue(newDateStatsData.getHighValue()); - } - if (newDateStatsData.isSetLowValue()) { - oldDateStatsData.setLowValue(newDateStatsData.getLowValue()); - } - if (newDateStatsData.isSetNumNulls()) { - oldDateStatsData.setNumNulls(newDateStatsData.getNumNulls()); - } - if (newDateStatsData.isSetNumDVs()) { - oldDateStatsData.setNumDVs(newDateStatsData.getNumDVs()); - } - if (newDateStatsData.isSetBitVectors()) { - oldDateStatsData.setBitVectors(newDateStatsData.getBitVectors()); - } - break; - } - default: - throw new IllegalArgumentException("Unknown stats type: " + typeNew.toString()); - } - } -}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java deleted file mode 100644 index 1a3ba3d..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TServerSocketKeepAlive.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import java.net.SocketException; - -import org.apache.thrift.transport.TServerSocket; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransportException; - -/** - * TServerSocketKeepAlive - like TServerSocket, but will enable keepalive for - * accepted sockets. - * - */ -public class TServerSocketKeepAlive extends TServerSocket { - public TServerSocketKeepAlive(TServerSocket serverSocket) throws TTransportException { - super(serverSocket.getServerSocket()); - } - - @Override - protected TSocket acceptImpl() throws TTransportException { - TSocket ts = super.acceptImpl(); - try { - ts.getSocket().setKeepAlive(true); - } catch (SocketException e) { - throw new TTransportException(e); - } - return ts; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java deleted file mode 100644 index c0c9604..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TSetIpAddressProcessor.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import java.lang.reflect.InvocationTargetException; -import java.net.Socket; - -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; -import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; -import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransport; - -/** - * TSetIpAddressProcessor passes the IP address of the Thrift client to the HMSHandler. - */ -public class TSetIpAddressProcessor<I extends Iface> extends ThriftHiveMetastore.Processor<Iface> { - - @SuppressWarnings("unchecked") - public TSetIpAddressProcessor(I iface) throws SecurityException, NoSuchFieldException, - IllegalArgumentException, IllegalAccessException, NoSuchMethodException, - InvocationTargetException { - super(iface); - } - - @Override - public boolean process(final TProtocol in, final TProtocol out) throws TException { - setIpAddress(in); - - return super.process(in, out); - } - - protected void setIpAddress(final TProtocol in) { - TTransport transport = in.getTransport(); - if (!(transport instanceof TSocket)) { - return; - } - setIpAddress(((TSocket)transport).getSocket()); - } - - protected void setIpAddress(final Socket inSocket) { - HMSHandler.setThreadLocalIpAddress(inSocket.getInetAddress().getHostAddress()); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java deleted file mode 100644 index 5285b54..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TUGIBasedProcessor.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.net.Socket; -import java.security.PrivilegedExceptionAction; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; -import org.apache.hadoop.hive.metastore.security.TUGIContainingTransport; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface; -import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.set_ugi_args; -import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.set_ugi_result; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.thrift.ProcessFunction; -import org.apache.thrift.TApplicationException; -import org.apache.thrift.TBase; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TMessage; -import org.apache.thrift.protocol.TMessageType; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TProtocolException; -import org.apache.thrift.protocol.TProtocolUtil; -import org.apache.thrift.protocol.TType; - -/** TUGIBasedProcessor is used in unsecure mode for thrift metastore client server communication. - * This processor checks whether the first rpc call after connection is set up is set_ugi() - * through which client sends ugi to server. Processor then perform all subsequent rpcs on the - * connection using ugi.doAs() so all actions are performed in client user context. - * Note that old clients will never call set_ugi() and thus ugi will never be received on server - * side, in which case server exhibits previous behavior and continues as usual. - */ -@SuppressWarnings("rawtypes") -public class TUGIBasedProcessor<I extends Iface> extends TSetIpAddressProcessor<Iface> { - - private final I iface; - private final Map<String, org.apache.thrift.ProcessFunction<Iface, ? extends TBase>> - functions; - static final Logger LOG = LoggerFactory.getLogger(TUGIBasedProcessor.class); - - public TUGIBasedProcessor(I iface) throws SecurityException, NoSuchFieldException, - IllegalArgumentException, IllegalAccessException, NoSuchMethodException, - InvocationTargetException { - super(iface); - this.iface = iface; - this.functions = getProcessMapView(); - } - - @SuppressWarnings("unchecked") - @Override - public boolean process(final TProtocol in, final TProtocol out) throws TException { - setIpAddress(in); - - final TMessage msg = in.readMessageBegin(); - final ProcessFunction<Iface, ? extends TBase> fn = functions.get(msg.name); - if (fn == null) { - TProtocolUtil.skip(in, TType.STRUCT); - in.readMessageEnd(); - TApplicationException x = new TApplicationException(TApplicationException.UNKNOWN_METHOD, - "Invalid method name: '"+msg.name+"'"); - out.writeMessageBegin(new TMessage(msg.name, TMessageType.EXCEPTION, msg.seqid)); - x.write(out); - out.writeMessageEnd(); - out.getTransport().flush(); - return true; - } - TUGIContainingTransport ugiTrans = (TUGIContainingTransport)in.getTransport(); - // Store ugi in transport if the rpc is set_ugi - if (msg.name.equalsIgnoreCase("set_ugi")){ - try { - handleSetUGI(ugiTrans, (ThriftHiveMetastore.Processor.set_ugi<Iface>)fn, msg, in, out); - } catch (TException e) { - throw e; - } catch (Exception e) { - throw new TException(e.getCause()); - } - return true; - } - UserGroupInformation clientUgi = ugiTrans.getClientUGI(); - if (null == clientUgi){ - // At this point, transport must contain client ugi, if it doesn't then its an old client. - fn.process(msg.seqid, in, out, iface); - return true; - } else { // Found ugi, perform doAs(). - PrivilegedExceptionAction<Void> pvea = new PrivilegedExceptionAction<Void>() { - @Override - public Void run() { - try { - fn.process(msg.seqid,in, out, iface); - return null; - } catch (TException te) { - throw new RuntimeException(te); - } - } - }; - try { - clientUgi.doAs(pvea); - return true; - } catch (RuntimeException rte) { - if (rte.getCause() instanceof TException) { - throw (TException)rte.getCause(); - } - throw rte; - } catch (InterruptedException ie) { - throw new RuntimeException(ie); // unexpected! - } catch (IOException ioe) { - throw new RuntimeException(ioe); // unexpected! - } finally { - try { - FileSystem.closeAllForUGI(clientUgi); - } catch (IOException e) { - LOG.error("Could not clean up file-system handles for UGI: " + clientUgi, e); - } - } - } - } - - private void handleSetUGI(TUGIContainingTransport ugiTrans, - ThriftHiveMetastore.Processor.set_ugi<Iface> fn, TMessage msg, TProtocol iprot, TProtocol oprot) - throws TException, SecurityException, NoSuchMethodException, IllegalArgumentException, - IllegalAccessException, InvocationTargetException{ - - UserGroupInformation clientUgi = ugiTrans.getClientUGI(); - if( null != clientUgi){ - throw new TException(new IllegalStateException("UGI is already set. Resetting is not " + - "allowed. Current ugi is: " + clientUgi.getUserName())); - } - - set_ugi_args args = fn.getEmptyArgsInstance(); - try { - args.read(iprot); - } catch (TProtocolException e) { - iprot.readMessageEnd(); - TApplicationException x = new TApplicationException(TApplicationException.PROTOCOL_ERROR, - e.getMessage()); - oprot.writeMessageBegin(new TMessage(msg.name, TMessageType.EXCEPTION, msg.seqid)); - x.write(oprot); - oprot.writeMessageEnd(); - oprot.getTransport().flush(); - return; - } - iprot.readMessageEnd(); - set_ugi_result result = fn.getResult(iface, args); - List<String> principals = result.getSuccess(); - // Store the ugi in transport and then continue as usual. - ugiTrans.setClientUGI(UserGroupInformation.createRemoteUser(principals.remove(principals.size()-1))); - oprot.writeMessageBegin(new TMessage(msg.name, TMessageType.REPLY, msg.seqid)); - result.write(oprot); - oprot.writeMessageEnd(); - oprot.getTransport().flush(); - } - - @Override - protected void setIpAddress(final TProtocol in) { - TUGIContainingTransport ugiTrans = (TUGIContainingTransport)in.getTransport(); - Socket socket = ugiTrans.getSocket(); - if (socket != null) { - setIpAddress(socket); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java deleted file mode 100644 index 1a17fe3..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TableIterable.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.thrift.TException; - -/** - * Use this to get Table objects for a table list. It provides an iterator to - * on the resulting Table objects. It batches the calls to - * IMetaStoreClient.getTableObjectsByName to avoid OOM issues in HS2 (with - * embedded metastore) or MetaStore server (if HS2 is using remote metastore). - * - */ -public class TableIterable implements Iterable<Table> { - - @Override - public Iterator<Table> iterator() { - return new Iterator<Table>() { - - private final Iterator<String> tableNamesIter = tableNames.iterator(); - private Iterator<org.apache.hadoop.hive.metastore.api.Table> batchIter = null; - - @Override - public boolean hasNext() { - return ((batchIter != null) && batchIter.hasNext()) || tableNamesIter.hasNext(); - } - - @Override - public Table next() { - if ((batchIter == null) || !batchIter.hasNext()) { - getNextBatch(); - } - return batchIter.next(); - } - - private void getNextBatch() { - // get next batch of table names in this list - List<String> nameBatch = new ArrayList<String>(); - int batchCounter = 0; - while (batchCounter < batchSize && tableNamesIter.hasNext()) { - nameBatch.add(tableNamesIter.next()); - batchCounter++; - } - // get the Table objects for this batch of table names and get iterator - // on it - - try { - if (catName != null) { - batchIter = msc.getTableObjectsByName(catName, dbname, nameBatch).iterator(); - } else { - batchIter = msc.getTableObjectsByName(dbname, nameBatch).iterator(); - } - } catch (TException e) { - throw new RuntimeException(e); - } - - } - - @Override - public void remove() { - throw new IllegalStateException( - "TableIterable is a read-only iterable and remove() is unsupported"); - } - }; - } - - private final IMetaStoreClient msc; - private final String dbname; - private final List<String> tableNames; - private final int batchSize; - private final String catName; - - /** - * Primary constructor that fetches all tables in a given msc, given a Hive - * object,a db name and a table name list. - */ - public TableIterable(IMetaStoreClient msc, String dbname, List<String> tableNames, int batchSize) - throws TException { - this.msc = msc; - this.catName = null; - this.dbname = dbname; - this.tableNames = tableNames; - this.batchSize = batchSize; - } - - public TableIterable(IMetaStoreClient msc, String catName, String dbname, List<String> - tableNames, int batchSize) throws TException { - this.msc = msc; - this.catName = catName; - this.dbname = dbname; - this.tableNames = tableNames; - this.batchSize = batchSize; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ThreadPool.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ThreadPool.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ThreadPool.java deleted file mode 100644 index d0fcd25..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ThreadPool.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - -/** - * Utility singleton class to manage all the threads. - */ -public class ThreadPool { - - static final private Logger LOG = LoggerFactory.getLogger(ThreadPool.class); - private static ThreadPool self = null; - private static ScheduledExecutorService pool; - - public static synchronized ThreadPool initialize(Configuration conf) { - if (self == null) { - self = new ThreadPool(conf); - LOG.debug("ThreadPool initialized"); - } - return self; - } - - private ThreadPool(Configuration conf) { - pool = Executors.newScheduledThreadPool(MetastoreConf.getIntVar(conf, - MetastoreConf.ConfVars.THREAD_POOL_SIZE)); - } - - public static ScheduledExecutorService getPool() { - if (self == null) { - throw new RuntimeException("ThreadPool accessed before initialized"); - } - return pool; - } - - public static synchronized void shutdown() { - if (self != null) { - pool.shutdown(); - self = null; - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java deleted file mode 100644 index b6a67ac..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/TransactionalMetaStoreEventListener.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.conf.Configuration; - -/** - * Special type of MetaStoreEventListener which should only be called in a transactional context - * and only if the transaction is successful. - * The events are expected to have a success status. - */ -public abstract class TransactionalMetaStoreEventListener extends MetaStoreEventListener { - - /** - * Constructor - * - * @param config configuration object - */ - public TransactionalMetaStoreEventListener(Configuration config) { - super(config); - } - -}
