http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git
a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 96c8871..5354e70 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -126,7 +126,6 @@ public class MetaStoreUtils {
// configuration parameter documentation
// HIVE_SUPPORT_SPECICAL_CHARACTERS_IN_TABLE_NAMES in HiveConf as well.
public static final char[] specialCharactersInTableNames = new char[] { '/'
};
- final static Charset ENCODING = StandardCharsets.UTF_8;
public static Table createColumnsetSchema(String name, List<String> columns,
List<String> partCols, Configuration conf) throws MetaException {
@@ -1183,15 +1182,6 @@ public class MetaStoreUtils {
return addCols(getSchemaWithoutCols(sd, tblsd, parameters, databaseName,
tableName, partitionKeys), tblsd.getCols());
}
- public static List<String> getColumnNamesForTable(Table table) {
- List<String> colNames = new ArrayList<String>();
- Iterator<FieldSchema> colsIterator = table.getSd().getColsIterator();
- while (colsIterator.hasNext()) {
- colNames.add(colsIterator.next().getName());
- }
- return colNames;
- }
-
public static String getColumnNameDelimiter(List<FieldSchema> fieldSchemas) {
// we first take a look if any fieldSchemas contain COMMA
for (int i = 0; i < fieldSchemas.size(); i++) {
@@ -1792,15 +1782,6 @@ public class MetaStoreUtils {
/**
* convert Exception to MetaException, which sets the cause to such exception
- * @param e cause of the exception
- * @return the MetaException with the specified exception as the cause
- */
- public static MetaException newMetaException(Exception e) {
- return newMetaException(e != null ? e.getMessage() : null, e);
- }
-
- /**
- * convert Exception to MetaException, which sets the cause to such exception
* @param errorMessage the error message for this MetaException
* @param e cause of the exception
* @return the MetaException with the specified exception as the cause
@@ -1821,174 +1802,12 @@ public class MetaStoreUtils {
return cols;
}
-
- // given a list of partStats, this function will give you an aggr stats
- public static List<ColumnStatisticsObj>
aggrPartitionStats(List<ColumnStatistics> partStats,
- String dbName, String tableName, List<String> partNames, List<String>
colNames,
- boolean useDensityFunctionForNDVEstimation, double ndvTuner)
- throws MetaException {
- // 1. group by the stats by colNames
- // map the colName to List<ColumnStatistics>
- Map<String, List<ColumnStatistics>> map = new HashMap<>();
- for (ColumnStatistics css : partStats) {
- List<ColumnStatisticsObj> objs = css.getStatsObj();
- for (ColumnStatisticsObj obj : objs) {
- List<ColumnStatisticsObj> singleObj = new ArrayList<>();
- singleObj.add(obj);
- ColumnStatistics singleCS = new ColumnStatistics(css.getStatsDesc(),
singleObj);
- if (!map.containsKey(obj.getColName())) {
- map.put(obj.getColName(), new ArrayList<ColumnStatistics>());
- }
- map.get(obj.getColName()).add(singleCS);
- }
- }
- return
aggrPartitionStats(map,dbName,tableName,partNames,colNames,useDensityFunctionForNDVEstimation,
ndvTuner);
- }
-
- public static List<ColumnStatisticsObj> aggrPartitionStats(
- Map<String, List<ColumnStatistics>> map, String dbName, String tableName,
- final List<String> partNames, List<String> colNames,
- final boolean useDensityFunctionForNDVEstimation,final double ndvTuner)
throws MetaException {
- List<ColumnStatisticsObj> colStats = new ArrayList<>();
- // 2. Aggregate stats for each column in a separate thread
- if (map.size()< 1) {
- //stats are absent in RDBMS
- LOG.debug("No stats data found for: dbName=" +dbName +" tblName=" +
tableName +
- " partNames= " + partNames + " colNames=" + colNames );
- return colStats;
- }
- final ExecutorService pool =
Executors.newFixedThreadPool(Math.min(map.size(), 16),
- new
ThreadFactoryBuilder().setDaemon(true).setNameFormat("aggr-col-stats-%d").build());
- final List<Future<ColumnStatisticsObj>> futures = Lists.newLinkedList();
-
- long start = System.currentTimeMillis();
- for (final Entry<String, List<ColumnStatistics>> entry : map.entrySet()) {
- futures.add(pool.submit(new Callable<ColumnStatisticsObj>() {
- @Override
- public ColumnStatisticsObj call() throws Exception {
- List<ColumnStatistics> css = entry.getValue();
- ColumnStatsAggregator aggregator =
ColumnStatsAggregatorFactory.getColumnStatsAggregator(css
-
.iterator().next().getStatsObj().iterator().next().getStatsData().getSetField(),
- useDensityFunctionForNDVEstimation, ndvTuner);
- ColumnStatisticsObj statsObj = aggregator.aggregate(entry.getKey(),
partNames, css);
- return statsObj;
- }}));
- }
- pool.shutdown();
- for (Future<ColumnStatisticsObj> future : futures) {
- try {
- colStats.add(future.get());
- } catch (InterruptedException | ExecutionException e) {
- pool.shutdownNow();
- LOG.debug(e.toString());
- throw new MetaException(e.toString());
- }
- }
- LOG.debug("Time for aggr col stats in seconds: {} Threads used: {}",
- ((System.currentTimeMillis() - (double)start))/1000,
Math.min(map.size(), 16));
- return colStats;
- }
-
-
- /**
- * Produce a hash for the storage descriptor
- * @param sd storage descriptor to hash
- * @param md message descriptor to use to generate the hash
- * @return the hash as a byte array
- */
- public static byte[] hashStorageDescriptor(StorageDescriptor sd,
MessageDigest md) {
- // Note all maps and lists have to be absolutely sorted. Otherwise we'll
produce different
- // results for hashes based on the OS or JVM being used.
- md.reset();
- for (FieldSchema fs : sd.getCols()) {
- md.update(fs.getName().getBytes(ENCODING));
- md.update(fs.getType().getBytes(ENCODING));
- if (fs.getComment() != null) {
- md.update(fs.getComment().getBytes(ENCODING));
- }
- }
- if (sd.getInputFormat() != null) {
- md.update(sd.getInputFormat().getBytes(ENCODING));
- }
- if (sd.getOutputFormat() != null) {
- md.update(sd.getOutputFormat().getBytes(ENCODING));
- }
- md.update(sd.isCompressed() ? "true".getBytes(ENCODING) :
"false".getBytes(ENCODING));
- md.update(Integer.toString(sd.getNumBuckets()).getBytes(ENCODING));
- if (sd.getSerdeInfo() != null) {
- SerDeInfo serde = sd.getSerdeInfo();
- if (serde.getName() != null) {
- md.update(serde.getName().getBytes(ENCODING));
- }
- if (serde.getSerializationLib() != null) {
- md.update(serde.getSerializationLib().getBytes(ENCODING));
- }
- if (serde.getParameters() != null) {
- SortedMap<String, String> params = new
TreeMap<>(serde.getParameters());
- for (Entry<String, String> param : params.entrySet()) {
- md.update(param.getKey().getBytes(ENCODING));
- md.update(param.getValue().getBytes(ENCODING));
- }
- }
- }
- if (sd.getBucketCols() != null) {
- List<String> bucketCols = new ArrayList<>(sd.getBucketCols());
- for (String bucket : bucketCols) {
- md.update(bucket.getBytes(ENCODING));
- }
- }
- if (sd.getSortCols() != null) {
- SortedSet<Order> orders = new TreeSet<>(sd.getSortCols());
- for (Order order : orders) {
- md.update(order.getCol().getBytes(ENCODING));
- md.update(Integer.toString(order.getOrder()).getBytes(ENCODING));
- }
- }
- if (sd.getSkewedInfo() != null) {
- SkewedInfo skewed = sd.getSkewedInfo();
- if (skewed.getSkewedColNames() != null) {
- SortedSet<String> colnames = new TreeSet<>(skewed.getSkewedColNames());
- for (String colname : colnames) {
- md.update(colname.getBytes(ENCODING));
- }
- }
- if (skewed.getSkewedColValues() != null) {
- SortedSet<String> sortedOuterList = new TreeSet<>();
- for (List<String> innerList : skewed.getSkewedColValues()) {
- SortedSet<String> sortedInnerList = new TreeSet<>(innerList);
- sortedOuterList.add(StringUtils.join(sortedInnerList, "."));
- }
- for (String colval : sortedOuterList) {
- md.update(colval.getBytes(ENCODING));
- }
- }
- if (skewed.getSkewedColValueLocationMaps() != null) {
- SortedMap<String, String> sortedMap = new TreeMap<>();
- for (Entry<List<String>, String> smap :
skewed.getSkewedColValueLocationMaps().entrySet()) {
- SortedSet<String> sortedKey = new TreeSet<>(smap.getKey());
- sortedMap.put(StringUtils.join(sortedKey, "."), smap.getValue());
- }
- for (Entry<String, String> e : sortedMap.entrySet()) {
- md.update(e.getKey().getBytes(ENCODING));
- md.update(e.getValue().getBytes(ENCODING));
- }
- }
- md.update(sd.isStoredAsSubDirectories() ? "true".getBytes(ENCODING) :
"false".getBytes(ENCODING));
- }
-
- return md.digest();
- }
-
- public static double decimalToDouble(Decimal decimal) {
- return new BigDecimal(new BigInteger(decimal.getUnscaled()),
decimal.getScale()).doubleValue();
- }
-
/**
* Verify if the user is allowed to make DB notification related calls.
* Only the superusers defined in the Hadoop proxy user settings have the
permission.
*
* @param user the short user name
- * @param config that contains the proxy user settings
+ * @param conf that contains the proxy user settings
* @return if the user has the permission
*/
public static boolean checkUserHasHostProxyPrivileges(String user,
Configuration conf, String ipAddress) {
