http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java index 1636d48..80cb1de 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MaterializationsInvalidationCache.java @@ -130,10 +130,13 @@ public final class MaterializationsInvalidationCache { public void run() { try { RawStore store = handler.getMS(); - for (String dbName : store.getAllDatabases()) { - for (Table mv : store.getTableObjectsByName(dbName, store.getTables(dbName, null, TableType.MATERIALIZED_VIEW))) { - addMaterializedView(mv.getDbName(), mv.getTableName(), ImmutableSet.copyOf(mv.getCreationMetadata().getTablesUsed()), - mv.getCreationMetadata().getValidTxnList(), OpType.LOAD); + for (String catName : store.getCatalogs()) { + for (String dbName : store.getAllDatabases(catName)) { + for (Table mv : store.getTableObjectsByName(catName, dbName, + store.getTables(catName, dbName, null, TableType.MATERIALIZED_VIEW))) { + addMaterializedView(mv.getDbName(), mv.getTableName(), ImmutableSet.copyOf(mv.getCreationMetadata().getTablesUsed()), + mv.getCreationMetadata().getValidTxnList(), OpType.LOAD); + } } } LOG.info("Initialized materializations invalidation cache");
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 0dcf117..997f5fd 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -19,7 +19,9 @@ package org.apache.hadoop.hive.metastore; import static org.apache.commons.lang.StringUtils.join; +import static org.apache.commons.lang.StringUtils.normalizeSpace; import static org.apache.commons.lang.StringUtils.repeat; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; import java.sql.Blob; import java.sql.Clob; @@ -311,17 +313,19 @@ class MetaStoreDirectSql { } } - public Database getDatabase(String dbName) throws MetaException{ + public Database getDatabase(String catName, String dbName) throws MetaException{ Query queryDbSelector = null; Query queryDbParams = null; try { dbName = dbName.toLowerCase(); + catName = catName.toLowerCase(); String queryTextDbSelector= "select " + "\"DB_ID\", \"NAME\", \"DB_LOCATION_URI\", \"DESC\", " - + "\"OWNER_NAME\", \"OWNER_TYPE\" " - + "FROM "+ DBS +" where \"NAME\" = ? "; - Object[] params = new Object[] { dbName }; + + "\"OWNER_NAME\", \"OWNER_TYPE\", \"CTLG_NAME\" " + + "FROM "+ DBS + + " where \"NAME\" = ? and \"CTLG_NAME\" = ? "; + Object[] params = new Object[] { dbName, catName }; queryDbSelector = pm.newQuery("javax.jdo.query.SQL", queryTextDbSelector); if (LOG.isTraceEnabled()) { @@ -370,6 +374,7 @@ class MetaStoreDirectSql { String type = extractSqlString(dbline[5]); db.setOwnerType( (null == type || type.trim().isEmpty()) ? null : PrincipalType.valueOf(type)); + db.setCatalogName(extractSqlString(dbline[6])); db.setParameters(MetaStoreUtils.trimMapNulls(dbParams,convertMapNullsToEmptyStrings)); if (LOG.isDebugEnabled()){ LOG.debug("getDatabase: directsql returning db " + db.getName() @@ -389,20 +394,22 @@ class MetaStoreDirectSql { /** * Get table names by using direct SQL queries. - * + * @param catName catalog name * @param dbName Metastore database namme * @param tableType Table type, or null if we want to get all tables * @return list of table names */ - public List<String> getTables(String dbName, TableType tableType) throws MetaException { + public List<String> getTables(String catName, String dbName, TableType tableType) + throws MetaException { String queryText = "SELECT " + TBLS + ".\"TBL_NAME\"" + " FROM " + TBLS + " " + " INNER JOIN " + DBS + " ON " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" " - + " WHERE " + DBS + ".\"NAME\" = ? " + + " WHERE " + DBS + ".\"NAME\" = ? AND " + DBS + ".\"CTLG_NAME\" = ? " + (tableType == null ? "" : "AND " + TBLS + ".\"TBL_TYPE\" = ? ") ; - List<String> pms = new ArrayList<String>(); + List<String> pms = new ArrayList<>(); pms.add(dbName); + pms.add(catName); if (tableType != null) { pms.add(tableType.toString()); } @@ -436,13 +443,15 @@ class MetaStoreDirectSql { /** * Gets partitions by using direct SQL queries. * Note that batching is not needed for this method - list of names implies the batch size; + * @param catName Metastore catalog name. * @param dbName Metastore db name. * @param tblName Metastore table name. * @param partNames Partition names to get. * @return List of partitions. */ - public List<Partition> getPartitionsViaSqlFilter(final String dbName, final String tblName, - List<String> partNames) throws MetaException { + public List<Partition> getPartitionsViaSqlFilter(final String catName, final String dbName, + final String tblName, List<String> partNames) + throws MetaException { if (partNames.isEmpty()) { return Collections.emptyList(); } @@ -450,7 +459,7 @@ class MetaStoreDirectSql { @Override public List<Partition> run(List<String> input) throws MetaException { String filter = "" + PARTITIONS + ".\"PART_NAME\" in (" + makeParams(input.size()) + ")"; - return getPartitionsViaSqlFilterInternal(dbName, tblName, null, filter, input, + return getPartitionsViaSqlFilterInternal(catName, dbName, tblName, null, filter, input, Collections.<String>emptyList(), null); } }); @@ -465,13 +474,15 @@ class MetaStoreDirectSql { public List<Partition> getPartitionsViaSqlFilter( SqlFilterForPushdown filter, Integer max) throws MetaException { Boolean isViewTable = isViewTable(filter.table); - return getPartitionsViaSqlFilterInternal(filter.table.getDbName(), filter.table.getTableName(), - isViewTable, filter.filter, filter.params, filter.joins, max); + String catName = filter.table.isSetCatName() ? filter.table.getCatName() : + DEFAULT_CATALOG_NAME; + return getPartitionsViaSqlFilterInternal(catName, filter.table.getDbName(), + filter.table.getTableName(), isViewTable, filter.filter, filter.params, filter.joins, max); } public static class SqlFilterForPushdown { - private final List<Object> params = new ArrayList<Object>(); - private final List<String> joins = new ArrayList<String>(); + private final List<Object> params = new ArrayList<>(); + private final List<String> joins = new ArrayList<>(); private String filter; private Table table; } @@ -488,14 +499,15 @@ class MetaStoreDirectSql { /** * Gets all partitions of a table by using direct SQL queries. + * @param catName Metastore catalog name. * @param dbName Metastore db name. * @param tblName Metastore table name. * @param max The maximum number of partitions to return. * @return List of partitions. */ - public List<Partition> getPartitions( + public List<Partition> getPartitions(String catName, String dbName, String tblName, Integer max) throws MetaException { - return getPartitionsViaSqlFilterInternal(dbName, tblName, null, + return getPartitionsViaSqlFilterInternal(catName, dbName, tblName, null, null, Collections.<String>emptyList(), Collections.<String>emptyList(), max); } @@ -504,13 +516,13 @@ class MetaStoreDirectSql { t.getTableType().equals(TableType.VIRTUAL_VIEW.toString()) : null; } - private boolean isViewTable(String dbName, String tblName) throws MetaException { + private boolean isViewTable(String catName, String dbName, String tblName) throws MetaException { Query query = null; try { String queryText = "select \"TBL_TYPE\" from " + TBLS + "" + " inner join " + DBS + " on " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" " + - " where " + TBLS + ".\"TBL_NAME\" = ? and " + DBS + ".\"NAME\" = ?"; - Object[] params = new Object[] { tblName, dbName }; + " where " + TBLS + ".\"TBL_NAME\" = ? and " + DBS + ".\"NAME\" = ? and " + DBS + ".\"CTLG_NAME\" = ?"; + Object[] params = new Object[] { tblName, dbName, catName }; query = pm.newQuery("javax.jdo.query.SQL", queryText); query.setUnique(true); Object result = executeWithArray(query, params, queryText); @@ -536,11 +548,13 @@ class MetaStoreDirectSql { * @param max The maximum number of partitions to return. * @return List of partition objects. */ - private List<Partition> getPartitionsViaSqlFilterInternal(String dbName, String tblName, - final Boolean isView, String sqlFilter, List<? extends Object> paramsForFilter, - List<String> joinsForFilter, Integer max) throws MetaException { + private List<Partition> getPartitionsViaSqlFilterInternal( + String catName, String dbName, String tblName, final Boolean isView, String sqlFilter, + List<? extends Object> paramsForFilter, List<String> joinsForFilter,Integer max) + throws MetaException { boolean doTrace = LOG.isDebugEnabled(); final String dbNameLcase = dbName.toLowerCase(), tblNameLcase = tblName.toLowerCase(); + final String catNameLcase = normalizeSpace(catName); // We have to be mindful of order during filtering if we are not returning all partitions. String orderForFilter = (max != null) ? " order by \"PART_NAME\" asc" : ""; @@ -559,12 +573,14 @@ class MetaStoreDirectSql { + " inner join " + DBS + " on " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" " + " and " + DBS + ".\"NAME\" = ? " + join(joinsForFilter, ' ') - + (StringUtils.isBlank(sqlFilter) ? "" : (" where " + sqlFilter)) + orderForFilter; - Object[] params = new Object[paramsForFilter.size() + 2]; + + " where " + DBS + ".\"CTLG_NAME\" = ? " + + (StringUtils.isBlank(sqlFilter) ? "" : (" and " + sqlFilter)) + orderForFilter; + Object[] params = new Object[paramsForFilter.size() + 3]; params[0] = tblNameLcase; params[1] = dbNameLcase; + params[2] = catNameLcase; for (int i = 0; i < paramsForFilter.size(); ++i) { - params[i + 2] = paramsForFilter.get(i); + params[i + 3] = paramsForFilter.get(i); } long start = doTrace ? System.nanoTime() : 0; @@ -583,7 +599,8 @@ class MetaStoreDirectSql { List<Partition> result = runBatched(sqlResult, new Batchable<Object, Partition>() { @Override public List<Partition> run(List<Object> input) throws MetaException { - return getPartitionsFromPartitionIds(dbNameLcase, tblNameLcase, isView, input); + return getPartitionsFromPartitionIds(catNameLcase, dbNameLcase, tblNameLcase, isView, + input); } }); @@ -592,7 +609,7 @@ class MetaStoreDirectSql { } /** Should be called with the list short enough to not trip up Oracle/etc. */ - private List<Partition> getPartitionsFromPartitionIds(String dbName, String tblName, + private List<Partition> getPartitionsFromPartitionIds(String catName, String dbName, String tblName, Boolean isView, List<Object> partIdList) throws MetaException { boolean doTrace = LOG.isDebugEnabled(); int idStringWidth = (int)Math.ceil(Math.log10(partIdList.size())) + 1; // 1 for comma @@ -635,6 +652,7 @@ class MetaStoreDirectSql { StringBuilder colsSb = new StringBuilder(7); // We expect that there's only one field schema. tblName = tblName.toLowerCase(); dbName = dbName.toLowerCase(); + catName = catName.toLowerCase(); for (Object[] fields : sqlResult) { // Here comes the ugly part... long partitionId = extractSqlLong(fields[0]); @@ -644,7 +662,7 @@ class MetaStoreDirectSql { // A partition must have at least sdId and serdeId set, or nothing set if it's a view. if (sdId == null || serdeId == null) { if (isView == null) { - isView = isViewTable(dbName, tblName); + isView = isViewTable(catName, dbName, tblName); } if ((sdId != null || colId != null || serdeId != null) || !isView) { throw new MetaException("Unexpected null for one of the IDs, SD " + sdId + @@ -655,8 +673,9 @@ class MetaStoreDirectSql { Partition part = new Partition(); orderedResult.add(part); // Set the collection fields; some code might not check presence before accessing them. - part.setParameters(new HashMap<String, String>()); + part.setParameters(new HashMap<>()); part.setValues(new ArrayList<String>()); + part.setCatName(catName); part.setDbName(dbName); part.setTableName(tblName); if (fields[4] != null) part.setCreateTime(extractSqlInt(fields[4])); @@ -910,6 +929,7 @@ class MetaStoreDirectSql { public int getNumPartitionsViaSqlFilter(SqlFilterForPushdown filter) throws MetaException { boolean doTrace = LOG.isDebugEnabled(); + String catName = filter.table.getCatName().toLowerCase(); String dbName = filter.table.getDbName().toLowerCase(); String tblName = filter.table.getTableName().toLowerCase(); @@ -920,13 +940,15 @@ class MetaStoreDirectSql { + " inner join " + DBS + " on " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" " + " and " + DBS + ".\"NAME\" = ? " + join(filter.joins, ' ') - + (filter.filter == null || filter.filter.trim().isEmpty() ? "" : (" where " + filter.filter)); + + " where " + DBS + ".\"CTLG_NAME\" = ? " + + (filter.filter == null || filter.filter.trim().isEmpty() ? "" : (" and " + filter.filter)); - Object[] params = new Object[filter.params.size() + 2]; + Object[] params = new Object[filter.params.size() + 3]; params[0] = tblName; params[1] = dbName; + params[2] = catName; for (int i = 0; i < filter.params.size(); ++i) { - params[i + 2] = filter.params.get(i); + params[i + 3] = filter.params.get(i); } long start = doTrace ? System.nanoTime() : 0; @@ -1291,10 +1313,12 @@ class MetaStoreDirectSql { if (dbHasJoinCastBug) { // This is a workaround for DERBY-6358 and Oracle bug; it is pretty horrible. tableValue += (" and " + TBLS + ".\"TBL_NAME\" = ? and " + DBS + ".\"NAME\" = ? and " + + DBS + ".\"CTLG_NAME\" = ? and " + "\"FILTER" + partColIndex + "\".\"PART_ID\" = " + PARTITIONS + ".\"PART_ID\" and " + "\"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex); params.add(table.getTableName().toLowerCase()); params.add(table.getDbName().toLowerCase()); + params.add(table.getCatName().toLowerCase()); } tableValue += " then " + tableValue0 + " else null end)"; } @@ -1311,29 +1335,32 @@ class MetaStoreDirectSql { /** * Retrieve the column statistics for the specified columns of the table. NULL * is returned if the columns are not provided. + * @param catName the catalog name of the table * @param dbName the database name of the table * @param tableName the table name * @param colNames the list of the column names * @return the column statistics for the specified columns * @throws MetaException */ - public ColumnStatistics getTableStats(final String dbName, final String tableName, - List<String> colNames, boolean enableBitVector) throws MetaException { + public ColumnStatistics getTableStats(final String catName, final String dbName, + final String tableName, List<String> colNames, + boolean enableBitVector) throws MetaException { if (colNames == null || colNames.isEmpty()) { return null; } final boolean doTrace = LOG.isDebugEnabled(); - final String queryText0 = "select " + getStatsList(enableBitVector) + " from " + TAB_COL_STATS + " " - + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in ("; + final String queryText0 = "select " + getStatsList(enableBitVector) + " from " + TAB_COL_STATS + + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in ("; Batchable<String, Object[]> b = new Batchable<String, Object[]>() { @Override public List<Object[]> run(List<String> input) throws MetaException { String queryText = queryText0 + makeParams(input.size()) + ")"; - Object[] params = new Object[input.size() + 2]; - params[0] = dbName; - params[1] = tableName; + Object[] params = new Object[input.size() + 3]; + params[0] = catName; + params[1] = dbName; + params[2] = tableName; for (int i = 0; i < input.size(); ++i) { - params[i + 2] = input.get(i); + params[i + 3] = input.get(i); } long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); @@ -1352,12 +1379,13 @@ class MetaStoreDirectSql { return null; } ColumnStatisticsDesc csd = new ColumnStatisticsDesc(true, dbName, tableName); + csd.setCatName(catName); ColumnStatistics result = makeColumnStats(list, csd, 0); b.closeAllQueries(); return result; } - public AggrStats aggrColStatsForPartitions(String dbName, String tableName, + public AggrStats aggrColStatsForPartitions(String catName, String dbName, String tableName, List<String> partNames, List<String> colNames, boolean useDensityFunctionForNDVEstimation, double ndvTuner, boolean enableBitVector) throws MetaException { if (colNames.isEmpty() || partNames.isEmpty()) { @@ -1379,33 +1407,33 @@ class MetaStoreDirectSql { boolean computePartsFound = true; for (String colName : colNames) { // Check the cache first - colStatsAggrCached = aggrStatsCache.get(dbName, tableName, colName, partNames); + colStatsAggrCached = aggrStatsCache.get(catName, dbName, tableName, colName, partNames); if (colStatsAggrCached != null) { colStatsList.add(colStatsAggrCached.getColStats()); partsFound = colStatsAggrCached.getNumPartsCached(); } else { if (computePartsFound) { - partsFound = partsFoundForPartitions(dbName, tableName, partNames, colNames); + partsFound = partsFoundForPartitions(catName, dbName, tableName, partNames, colNames); computePartsFound = false; } - List<String> colNamesForDB = new ArrayList<String>(); + List<String> colNamesForDB = new ArrayList<>(); colNamesForDB.add(colName); // Read aggregated stats for one column colStatsAggrFromDB = - columnStatisticsObjForPartitions(dbName, tableName, partNames, colNamesForDB, + columnStatisticsObjForPartitions(catName, dbName, tableName, partNames, colNamesForDB, partsFound, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector); if (!colStatsAggrFromDB.isEmpty()) { ColumnStatisticsObj colStatsAggr = colStatsAggrFromDB.get(0); colStatsList.add(colStatsAggr); // Update the cache to add this new aggregate node - aggrStatsCache.add(dbName, tableName, colName, partsFound, colStatsAggr, bloomFilter); + aggrStatsCache.add(catName, dbName, tableName, colName, partsFound, colStatsAggr, bloomFilter); } } } } else { - partsFound = partsFoundForPartitions(dbName, tableName, partNames, colNames); + partsFound = partsFoundForPartitions(catName, dbName, tableName, partNames, colNames); colStatsList = - columnStatisticsObjForPartitions(dbName, tableName, partNames, colNames, partsFound, + columnStatisticsObjForPartitions(catName, dbName, tableName, partNames, colNames, partsFound, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector); } LOG.info("useDensityFunctionForNDVEstimation = " + useDensityFunctionForNDVEstimation @@ -1423,12 +1451,13 @@ class MetaStoreDirectSql { return bloomFilter; } - private long partsFoundForPartitions(final String dbName, final String tableName, + private long partsFoundForPartitions( + final String catName, final String dbName, final String tableName, final List<String> partNames, List<String> colNames) throws MetaException { assert !colNames.isEmpty() && !partNames.isEmpty(); final boolean doTrace = LOG.isDebugEnabled(); final String queryText0 = "select count(\"COLUMN_NAME\") from " + PART_COL_STATS + "" - + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + " and \"COLUMN_NAME\" in (%1$s) and \"PARTITION_NAME\" in (%2$s)" + " group by \"PARTITION_NAME\""; List<Long> allCounts = runBatched(colNames, new Batchable<String, Long>() { @@ -1444,7 +1473,7 @@ class MetaStoreDirectSql { Query query = pm.newQuery("javax.jdo.query.SQL", queryText); try { Object qResult = executeWithArray(query, prepareParams( - dbName, tableName, inputPartNames, inputColName), queryText); + catName, dbName, tableName, inputPartNames, inputColName), queryText); long end = doTrace ? System.nanoTime() : 0; timingTrace(doTrace, queryText, start, end); ForwardQueryResult<?> fqr = (ForwardQueryResult<?>) qResult; @@ -1469,7 +1498,8 @@ class MetaStoreDirectSql { return partsFound; } - private List<ColumnStatisticsObj> columnStatisticsObjForPartitions(final String dbName, + private List<ColumnStatisticsObj> columnStatisticsObjForPartitions( + final String catName, final String dbName, final String tableName, final List<String> partNames, List<String> colNames, long partsFound, final boolean useDensityFunctionForNDVEstimation, final double ndvTuner, final boolean enableBitVector) throws MetaException { final boolean areAllPartsFound = (partsFound == partNames.size()); @@ -1479,7 +1509,7 @@ class MetaStoreDirectSql { return runBatched(partNames, new Batchable<String, ColumnStatisticsObj>() { @Override public List<ColumnStatisticsObj> run(List<String> inputPartNames) throws MetaException { - return columnStatisticsObjForPartitionsBatch(dbName, tableName, inputPartNames, + return columnStatisticsObjForPartitionsBatch(catName, dbName, tableName, inputPartNames, inputColNames, areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner, enableBitVector); } }); @@ -1487,10 +1517,10 @@ class MetaStoreDirectSql { }); } - public List<ColStatsObjWithSourceInfo> getColStatsForAllTablePartitions(String dbName, + public List<ColStatsObjWithSourceInfo> getColStatsForAllTablePartitions(String catName, String dbName, boolean enableBitVector) throws MetaException { String queryText = "select \"TABLE_NAME\", \"PARTITION_NAME\", " + getStatsList(enableBitVector) - + " from " + " " + PART_COL_STATS + " where \"DB_NAME\" = ?"; + + " from " + " " + PART_COL_STATS + " where \"DB_NAME\" = ? and \"CAT_NAME\" = ?"; long start = 0; long end = 0; Query query = null; @@ -1500,7 +1530,7 @@ class MetaStoreDirectSql { List<ColStatsObjWithSourceInfo> colStatsForDB = new ArrayList<ColStatsObjWithSourceInfo>(); try { query = pm.newQuery("javax.jdo.query.SQL", queryText); - qResult = executeWithArray(query, new Object[] { dbName }, queryText); + qResult = executeWithArray(query, new Object[] { dbName, catName }, queryText); if (qResult == null) { query.closeAll(); return colStatsForDB; @@ -1512,7 +1542,7 @@ class MetaStoreDirectSql { String tblName = (String) row[0]; String partName = (String) row[1]; ColumnStatisticsObj colStatObj = prepareCSObj(row, 2); - colStatsForDB.add(new ColStatsObjWithSourceInfo(colStatObj, dbName, tblName, partName)); + colStatsForDB.add(new ColStatsObjWithSourceInfo(colStatObj, catName, dbName, tblName, partName)); Deadline.checkTimeout(); } } finally { @@ -1522,31 +1552,31 @@ class MetaStoreDirectSql { } /** Should be called with the list short enough to not trip up Oracle/etc. */ - private List<ColumnStatisticsObj> columnStatisticsObjForPartitionsBatch(String dbName, + private List<ColumnStatisticsObj> columnStatisticsObjForPartitionsBatch(String catName, String dbName, String tableName, List<String> partNames, List<String> colNames, boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner, boolean enableBitVector) throws MetaException { if (enableBitVector) { - return aggrStatsUseJava(dbName, tableName, partNames, colNames, areAllPartsFound, + return aggrStatsUseJava(catName, dbName, tableName, partNames, colNames, areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner); } else { - return aggrStatsUseDB(dbName, tableName, partNames, colNames, areAllPartsFound, + return aggrStatsUseDB(catName, dbName, tableName, partNames, colNames, areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner); } } - private List<ColumnStatisticsObj> aggrStatsUseJava(String dbName, String tableName, + private List<ColumnStatisticsObj> aggrStatsUseJava(String catName, String dbName, String tableName, List<String> partNames, List<String> colNames, boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException { // 1. get all the stats for colNames in partNames; List<ColumnStatistics> partStats = - getPartitionStats(dbName, tableName, partNames, colNames, true); + getPartitionStats(catName, dbName, tableName, partNames, colNames, true); // 2. use util function to aggr stats - return MetaStoreUtils.aggrPartitionStats(partStats, dbName, tableName, partNames, colNames, + return MetaStoreUtils.aggrPartitionStats(partStats, catName, dbName, tableName, partNames, colNames, areAllPartsFound, useDensityFunctionForNDVEstimation, ndvTuner); } - private List<ColumnStatisticsObj> aggrStatsUseDB(String dbName, + private List<ColumnStatisticsObj> aggrStatsUseDB(String catName, String dbName, String tableName, List<String> partNames, List<String> colNames, boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException { // TODO: all the extrapolation logic should be moved out of this class, @@ -1573,7 +1603,7 @@ class MetaStoreDirectSql { + "avg((\"DOUBLE_HIGH_VALUE\"-\"DOUBLE_LOW_VALUE\")/\"NUM_DISTINCTS\")," + "avg((cast(\"BIG_DECIMAL_HIGH_VALUE\" as decimal)-cast(\"BIG_DECIMAL_LOW_VALUE\" as decimal))/\"NUM_DISTINCTS\")," + "sum(\"NUM_DISTINCTS\")" + " from " + PART_COL_STATS + "" - + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "; + + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "; String queryText = null; long start = 0; long end = 0; @@ -1589,7 +1619,7 @@ class MetaStoreDirectSql { + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); - qResult = executeWithArray(query, prepareParams(dbName, tableName, partNames, colNames), + qResult = executeWithArray(query, prepareParams(catName, dbName, tableName, partNames, colNames), queryText); if (qResult == null) { query.closeAll(); @@ -1612,13 +1642,13 @@ class MetaStoreDirectSql { List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(colNames.size()); queryText = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", count(\"PARTITION_NAME\") " + " from " + PART_COL_STATS - + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); - qResult = executeWithArray(query, prepareParams(dbName, tableName, partNames, colNames), + qResult = executeWithArray(query, prepareParams(catName, dbName, tableName, partNames, colNames), queryText); end = doTrace ? System.nanoTime() : 0; timingTrace(doTrace, queryText, start, end); @@ -1653,7 +1683,7 @@ class MetaStoreDirectSql { start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); qResult = executeWithArray(query, - prepareParams(dbName, tableName, partNames, noExtraColumnNames), queryText); + prepareParams(catName, dbName, tableName, partNames, noExtraColumnNames), queryText); if (qResult == null) { query.closeAll(); return Collections.emptyList(); @@ -1677,7 +1707,7 @@ class MetaStoreDirectSql { // get sum for all columns to reduce the number of queries Map<String, Map<Integer, Object>> sumMap = new HashMap<String, Map<Integer, Object>>(); queryText = "select \"COLUMN_NAME\", sum(\"NUM_NULLS\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\"), sum(\"NUM_DISTINCTS\")" - + " from " + PART_COL_STATS + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + + " from " + PART_COL_STATS + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + " and \"COLUMN_NAME\" in (" + makeParams(extraColumnNameTypeParts.size()) + ") and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ") group by \"COLUMN_NAME\""; @@ -1686,7 +1716,7 @@ class MetaStoreDirectSql { List<String> extraColumnNames = new ArrayList<String>(); extraColumnNames.addAll(extraColumnNameTypeParts.keySet()); qResult = executeWithArray(query, - prepareParams(dbName, tableName, partNames, extraColumnNames), queryText); + prepareParams(catName, dbName, tableName, partNames, extraColumnNames), queryText); if (qResult == null) { query.closeAll(); return Collections.emptyList(); @@ -1750,20 +1780,20 @@ class MetaStoreDirectSql { if (!decimal) { queryText = "select \"" + colStatName + "\",\"PARTITION_NAME\" from " + PART_COL_STATS - + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?" + + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + " order by \"" + colStatName + "\""; } else { queryText = "select \"" + colStatName + "\",\"PARTITION_NAME\" from " + PART_COL_STATS - + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?" + + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + " order by cast(\"" + colStatName + "\" as decimal)"; } start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); qResult = executeWithArray(query, - prepareParams(dbName, tableName, partNames, Arrays.asList(colName)), queryText); + prepareParams(catName, dbName, tableName, partNames, Arrays.asList(colName)), queryText); if (qResult == null) { query.closeAll(); return Collections.emptyList(); @@ -1786,13 +1816,13 @@ class MetaStoreDirectSql { + "avg((\"LONG_HIGH_VALUE\"-\"LONG_LOW_VALUE\")/cast(\"NUM_DISTINCTS\" as decimal))," + "avg((\"DOUBLE_HIGH_VALUE\"-\"DOUBLE_LOW_VALUE\")/\"NUM_DISTINCTS\")," + "avg((cast(\"BIG_DECIMAL_HIGH_VALUE\" as decimal)-cast(\"BIG_DECIMAL_LOW_VALUE\" as decimal))/\"NUM_DISTINCTS\")" - + " from " + PART_COL_STATS + "" + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + + " from " + PART_COL_STATS + "" + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + " and \"COLUMN_NAME\" = ?" + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + " group by \"COLUMN_NAME\""; start = doTrace ? System.nanoTime() : 0; query = pm.newQuery("javax.jdo.query.SQL", queryText); qResult = executeWithArray(query, - prepareParams(dbName, tableName, partNames, Arrays.asList(colName)), queryText); + prepareParams(catName, dbName, tableName, partNames, Arrays.asList(colName)), queryText); if (qResult == null) { query.closeAll(); return Collections.emptyList(); @@ -1837,11 +1867,12 @@ class MetaStoreDirectSql { return cso; } - private Object[] prepareParams(String dbName, String tableName, List<String> partNames, - List<String> colNames) throws MetaException { + private Object[] prepareParams(String catName, String dbName, String tableName, + List<String> partNames, List<String> colNames) throws MetaException { - Object[] params = new Object[colNames.size() + partNames.size() + 2]; + Object[] params = new Object[colNames.size() + partNames.size() + 3]; int paramI = 0; + params[paramI++] = catName; params[paramI++] = dbName; params[paramI++] = tableName; for (String colName : colNames) { @@ -1854,14 +1885,16 @@ class MetaStoreDirectSql { return params; } - public List<ColumnStatistics> getPartitionStats(final String dbName, final String tableName, - final List<String> partNames, List<String> colNames, boolean enableBitVector) throws MetaException { + public List<ColumnStatistics> getPartitionStats( + final String catName, final String dbName, final String tableName, final List<String> partNames, + List<String> colNames, boolean enableBitVector) throws MetaException { if (colNames.isEmpty() || partNames.isEmpty()) { return Collections.emptyList(); } final boolean doTrace = LOG.isDebugEnabled(); final String queryText0 = "select \"PARTITION_NAME\", " + getStatsList(enableBitVector) + " from " - + " " + PART_COL_STATS + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\"" + + " " + PART_COL_STATS + " where \"CAT_NAME\" = ? and \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and " + + "\"COLUMN_NAME\"" + " in (%1$s) AND \"PARTITION_NAME\" in (%2$s) order by \"PARTITION_NAME\""; Batchable<String, Object[]> b = new Batchable<String, Object[]>() { @Override @@ -1874,7 +1907,7 @@ class MetaStoreDirectSql { long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", queryText); Object qResult = executeWithArray(query, prepareParams( - dbName, tableName, inputPartNames, inputColNames), queryText); + catName, dbName, tableName, inputPartNames, inputColNames), queryText); timingTrace(doTrace, queryText0, start, (doTrace ? System.nanoTime() : 0)); if (qResult == null) { query.closeAll(); @@ -1904,6 +1937,7 @@ class MetaStoreDirectSql { continue; } else if (from != i) { ColumnStatisticsDesc csd = new ColumnStatisticsDesc(false, dbName, tableName); + csd.setCatName(catName); csd.setPartName(lastPartName); result.add(makeColumnStats(list.subList(from, i), csd, 1)); } @@ -2036,8 +2070,10 @@ class MetaStoreDirectSql { return result; } - public List<SQLForeignKey> getForeignKeys(String parent_db_name, String parent_tbl_name, String foreign_db_name, String foreign_tbl_name) throws MetaException { - List<SQLForeignKey> ret = new ArrayList<SQLForeignKey>(); + public List<SQLForeignKey> getForeignKeys(String catName, String parent_db_name, + String parent_tbl_name, String foreign_db_name, + String foreign_tbl_name) throws MetaException { + List<SQLForeignKey> ret = new ArrayList<>(); String queryText = "SELECT \"D2\".\"NAME\", \"T2\".\"TBL_NAME\", " + "CASE WHEN \"C2\".\"COLUMN_NAME\" IS NOT NULL THEN \"C2\".\"COLUMN_NAME\" " @@ -2065,6 +2101,7 @@ class MetaStoreDirectSql { + " \"P2\".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" " + " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = " + MConstraint.FOREIGN_KEY_CONSTRAINT + " AND \"KEY_CONSTRAINTS2\".\"CONSTRAINT_TYPE\" = " + MConstraint.PRIMARY_KEY_CONSTRAINT + " AND" + + " " + DBS + ".\"CTLG_NAME\" = ? AND" + (foreign_db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND") + (foreign_tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? AND") + (parent_tbl_name == null ? "" : " \"T2\".\"TBL_NAME\" = ? AND") @@ -2075,6 +2112,7 @@ class MetaStoreDirectSql { queryText = queryText.substring(0, queryText.length()-3); } List<String> pms = new ArrayList<String>(); + pms.add(catName); if (foreign_db_name != null) { pms.add(foreign_db_name); } @@ -2114,19 +2152,22 @@ class MetaStoreDirectSql { validate, rely ); - ret.add(currKey); + currKey.setCatName(catName); + ret.add(currKey); } } return ret; } - public List<SQLPrimaryKey> getPrimaryKeys(String db_name, String tbl_name) throws MetaException { - List<SQLPrimaryKey> ret = new ArrayList<SQLPrimaryKey>(); + public List<SQLPrimaryKey> getPrimaryKeys(String catName, String db_name, String tbl_name) + throws MetaException { + List<SQLPrimaryKey> ret = new ArrayList<>(); String queryText = "SELECT " + DBS + ".\"NAME\", " + TBLS + ".\"TBL_NAME\", " + "CASE WHEN " + COLUMNS_V2 + ".\"COLUMN_NAME\" IS NOT NULL THEN " + COLUMNS_V2 + ".\"COLUMN_NAME\" " + "ELSE " + PARTITION_KEYS + ".\"PKEY_NAME\" END, " + KEY_CONSTRAINTS + ".\"POSITION\", " - + "" + KEY_CONSTRAINTS + ".\"CONSTRAINT_NAME\", " + KEY_CONSTRAINTS + ".\"ENABLE_VALIDATE_RELY\" " + + KEY_CONSTRAINTS + ".\"CONSTRAINT_NAME\", " + KEY_CONSTRAINTS + ".\"ENABLE_VALIDATE_RELY\", " + + DBS + ".\"CTLG_NAME\"" + " from " + TBLS + " " + " INNER JOIN " + KEY_CONSTRAINTS + " ON " + TBLS + ".\"TBL_ID\" = " + KEY_CONSTRAINTS + ".\"PARENT_TBL_ID\" " + " INNER JOIN " + DBS + " ON " + TBLS + ".\"DB_ID\" = " + DBS + ".\"DB_ID\" " @@ -2135,6 +2176,7 @@ class MetaStoreDirectSql { + " LEFT OUTER JOIN " + PARTITION_KEYS + " ON " + TBLS + ".\"TBL_ID\" = " + PARTITION_KEYS + ".\"TBL_ID\" AND " + " " + PARTITION_KEYS + ".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" " + " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = "+ MConstraint.PRIMARY_KEY_CONSTRAINT + " AND" + + " " + DBS + ".\"CTLG_NAME\" = ? AND" + (db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND") + (tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? ") ; @@ -2142,7 +2184,8 @@ class MetaStoreDirectSql { if (queryText.endsWith("AND")) { queryText = queryText.substring(0, queryText.length()-3); } - List<String> pms = new ArrayList<String>(); + List<String> pms = new ArrayList<>(); + pms.add(catName); if (db_name != null) { pms.add(db_name); } @@ -2168,13 +2211,14 @@ class MetaStoreDirectSql { enable, validate, rely); - ret.add(currKey); + currKey.setCatName(extractSqlString(line[6])); + ret.add(currKey); } } return ret; } - public List<SQLUniqueConstraint> getUniqueConstraints(String db_name, String tbl_name) + public List<SQLUniqueConstraint> getUniqueConstraints(String catName, String db_name, String tbl_name) throws MetaException { List<SQLUniqueConstraint> ret = new ArrayList<SQLUniqueConstraint>(); String queryText = @@ -2190,6 +2234,7 @@ class MetaStoreDirectSql { + " LEFT OUTER JOIN " + PARTITION_KEYS + " ON " + TBLS + ".\"TBL_ID\" = " + PARTITION_KEYS + ".\"TBL_ID\" AND " + " " + PARTITION_KEYS + ".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" " + " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = "+ MConstraint.UNIQUE_CONSTRAINT + " AND" + + " " + DBS + ".\"CTLG_NAME\" = ? AND" + (db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND") + (tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? ") ; @@ -2198,6 +2243,7 @@ class MetaStoreDirectSql { queryText = queryText.substring(0, queryText.length()-3); } List<String> pms = new ArrayList<String>(); + pms.add(catName); if (db_name != null) { pms.add(db_name); } @@ -2215,23 +2261,23 @@ class MetaStoreDirectSql { boolean enable = (enableValidateRely & 4) != 0; boolean validate = (enableValidateRely & 2) != 0; boolean rely = (enableValidateRely & 1) != 0; - SQLUniqueConstraint currConstraint = new SQLUniqueConstraint( - extractSqlString(line[0]), - extractSqlString(line[1]), - extractSqlString(line[2]), - extractSqlInt(line[3]), extractSqlString(line[4]), - enable, - validate, - rely); - ret.add(currConstraint); + ret.add(new SQLUniqueConstraint( + catName, + extractSqlString(line[0]), + extractSqlString(line[1]), + extractSqlString(line[2]), + extractSqlInt(line[3]), extractSqlString(line[4]), + enable, + validate, + rely)); } } return ret; } - public List<SQLNotNullConstraint> getNotNullConstraints(String db_name, String tbl_name) + public List<SQLNotNullConstraint> getNotNullConstraints(String catName, String db_name, String tbl_name) throws MetaException { - List<SQLNotNullConstraint> ret = new ArrayList<SQLNotNullConstraint>(); + List<SQLNotNullConstraint> ret = new ArrayList<>(); String queryText = "SELECT " + DBS + ".\"NAME\", " + TBLS + ".\"TBL_NAME\"," + "CASE WHEN " + COLUMNS_V2 + ".\"COLUMN_NAME\" IS NOT NULL THEN " + COLUMNS_V2 + ".\"COLUMN_NAME\" " @@ -2245,6 +2291,7 @@ class MetaStoreDirectSql { + " LEFT OUTER JOIN " + PARTITION_KEYS + " ON " + TBLS + ".\"TBL_ID\" = " + PARTITION_KEYS + ".\"TBL_ID\" AND " + " " + PARTITION_KEYS + ".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" " + " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = "+ MConstraint.NOT_NULL_CONSTRAINT + " AND" + + " " + DBS + ".\"CTLG_NAME\" = ? AND" + (db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND") + (tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? ") ; @@ -2252,7 +2299,8 @@ class MetaStoreDirectSql { if (queryText.endsWith("AND")) { queryText = queryText.substring(0, queryText.length()-3); } - List<String> pms = new ArrayList<String>(); + List<String> pms = new ArrayList<>(); + pms.add(catName); if (db_name != null) { pms.add(db_name); } @@ -2270,21 +2318,21 @@ class MetaStoreDirectSql { boolean enable = (enableValidateRely & 4) != 0; boolean validate = (enableValidateRely & 2) != 0; boolean rely = (enableValidateRely & 1) != 0; - SQLNotNullConstraint currConstraint = new SQLNotNullConstraint( - extractSqlString(line[0]), - extractSqlString(line[1]), - extractSqlString(line[2]), - extractSqlString(line[3]), - enable, - validate, - rely); - ret.add(currConstraint); + ret.add(new SQLNotNullConstraint( + catName, + extractSqlString(line[0]), + extractSqlString(line[1]), + extractSqlString(line[2]), + extractSqlString(line[3]), + enable, + validate, + rely)); } } return ret; } - public List<SQLDefaultConstraint> getDefaultConstraints(String db_name, String tbl_name) + public List<SQLDefaultConstraint> getDefaultConstraints(String catName, String db_name, String tbl_name) throws MetaException { List<SQLDefaultConstraint> ret = new ArrayList<SQLDefaultConstraint>(); String queryText = @@ -2301,6 +2349,7 @@ class MetaStoreDirectSql { + " LEFT OUTER JOIN " + PARTITION_KEYS + " ON " + TBLS + ".\"TBL_ID\" = " + PARTITION_KEYS + ".\"TBL_ID\" AND " + " " + PARTITION_KEYS + ".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" " + " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = "+ MConstraint.DEFAULT_CONSTRAINT+ " AND" + + " " + DBS + ".\"CTLG_NAME\" = ? AND" + (db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND") + (tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? ") ; @@ -2311,7 +2360,8 @@ class MetaStoreDirectSql { if (LOG.isDebugEnabled()){ LOG.debug("getDefaultConstraints: directsql : " + queryText); } - List<String> pms = new ArrayList<String>(); + List<String> pms = new ArrayList<>(); + pms.add(catName); if (db_name != null) { pms.add(db_name); } @@ -2330,6 +2380,7 @@ class MetaStoreDirectSql { boolean validate = (enableValidateRely & 2) != 0; boolean rely = (enableValidateRely & 1) != 0; SQLDefaultConstraint currConstraint = new SQLDefaultConstraint( + catName, extractSqlString(line[0]), extractSqlString(line[1]), extractSqlString(line[2]), @@ -2344,7 +2395,7 @@ class MetaStoreDirectSql { return ret; } - public List<SQLCheckConstraint> getCheckConstraints(String db_name, String tbl_name) + public List<SQLCheckConstraint> getCheckConstraints(String catName, String db_name, String tbl_name) throws MetaException { List<SQLCheckConstraint> ret = new ArrayList<SQLCheckConstraint>(); String queryText = @@ -2361,6 +2412,7 @@ class MetaStoreDirectSql { + " LEFT OUTER JOIN " + PARTITION_KEYS + " ON " + TBLS + ".\"TBL_ID\" = " + PARTITION_KEYS + ".\"TBL_ID\" AND " + " " + PARTITION_KEYS + ".\"INTEGER_IDX\" = " + KEY_CONSTRAINTS + ".\"PARENT_INTEGER_IDX\" " + " WHERE " + KEY_CONSTRAINTS + ".\"CONSTRAINT_TYPE\" = "+ MConstraint.CHECK_CONSTRAINT+ " AND" + + " " + DBS + ".\"CTLG_NAME\" = ? AND" + (db_name == null ? "" : " " + DBS + ".\"NAME\" = ? AND") + (tbl_name == null ? "" : " " + TBLS + ".\"TBL_NAME\" = ? ") ; @@ -2371,7 +2423,8 @@ class MetaStoreDirectSql { if (LOG.isDebugEnabled()){ LOG.debug("getCheckConstraints: directsql : " + queryText); } - List<String> pms = new ArrayList<String>(); + List<String> pms = new ArrayList<>(); + pms.add(catName); if (db_name != null) { pms.add(db_name); } @@ -2390,6 +2443,7 @@ class MetaStoreDirectSql { boolean validate = (enableValidateRely & 2) != 0; boolean rely = (enableValidateRely & 1) != 0; SQLCheckConstraint currConstraint = new SQLCheckConstraint( + catName, extractSqlString(line[0]), extractSqlString(line[1]), extractSqlString(line[2]), http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java index 67600e1..569fff0 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java @@ -35,10 +35,12 @@ import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterSchemaVersionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; import org.apache.hadoop.hive.metastore.events.ConfigChangeEvent; +import org.apache.hadoop.hive.metastore.events.CreateCatalogEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent; import org.apache.hadoop.hive.metastore.events.CreateISchemaEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; +import org.apache.hadoop.hive.metastore.events.DropCatalogEvent; import org.apache.hadoop.hive.metastore.events.DropConstraintEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropFunctionEvent; @@ -222,6 +224,12 @@ public abstract class MetaStoreEventListener implements Configurable { throws MetaException { } + public void onCreateCatalog(CreateCatalogEvent createCatalogEvent) throws MetaException { + } + + public void onDropCatalog(DropCatalogEvent dropCatalogEvent) throws MetaException { + } + @Override public Configuration getConf() { return this.conf; http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java index 8522afe..f7a0cd0 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreFilterHook.java @@ -22,12 +22,14 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionSpec; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TableMeta; /** * Metadata filter hook for metastore client. This will be useful for authorization @@ -39,11 +41,31 @@ import org.apache.hadoop.hive.metastore.api.Table; public interface MetaStoreFilterHook { /** + * Filter a catalog object. Default implementation returns the passed in catalog. + * @param catalog catalog to filter + * @return filtered catalog + * @throws MetaException something bad happened + */ + default Catalog filterCatalog(Catalog catalog) throws MetaException { + return catalog; + } + + /** + * Filter a list of catalog names. Default implementation returns the passed in list. + * @param catalogs list of catalog names. + * @return filtered list of catalog names. + * @throws MetaException something bad happened. + */ + default List<String> filterCatalogs(List<String> catalogs) throws MetaException { + return catalogs; + } + + /** * Filter given list of databases * @param dbList * @return List of filtered Db names */ - public List<String> filterDatabases(List<String> dbList) throws MetaException; + List<String> filterDatabases(List<String> dbList) throws MetaException; /** * filter to given database object if applicable @@ -51,15 +73,27 @@ public interface MetaStoreFilterHook { * @return the same database if it's not filtered out * @throws NoSuchObjectException */ - public Database filterDatabase(Database dataBase) throws MetaException, NoSuchObjectException; + Database filterDatabase(Database dataBase) throws MetaException, NoSuchObjectException; /** * Filter given list of tables - * @param dbName - * @param tableList + * @param catName catalog name + * @param dbName database name + * @param tableList list of table returned by the metastore * @return List of filtered table names */ - public List<String> filterTableNames(String dbName, List<String> tableList) throws MetaException; + List<String> filterTableNames(String catName, String dbName, List<String> tableList) + throws MetaException; + + // Previously this was handled by filterTableNames. But it can't be anymore because we can no + // longer depend on a 1-1 mapping between table name and entry in the list. + /** + * Filter a list of TableMeta objects. + * @param tableMetas list of TableMetas to filter + * @return filtered table metas + * @throws MetaException something went wrong + */ + List<TableMeta> filterTableMetas(List<TableMeta> tableMetas) throws MetaException; /** * filter to given table object if applicable @@ -67,28 +101,28 @@ public interface MetaStoreFilterHook { * @return the same table if it's not filtered out * @throws NoSuchObjectException */ - public Table filterTable(Table table) throws MetaException, NoSuchObjectException; + Table filterTable(Table table) throws MetaException, NoSuchObjectException; /** * Filter given list of tables * @param tableList * @return List of filtered table names */ - public List<Table> filterTables(List<Table> tableList) throws MetaException; + List<Table> filterTables(List<Table> tableList) throws MetaException; /** * Filter given list of partitions * @param partitionList * @return */ - public List<Partition> filterPartitions(List<Partition> partitionList) throws MetaException; + List<Partition> filterPartitions(List<Partition> partitionList) throws MetaException; /** * Filter given list of partition specs * @param partitionSpecList * @return */ - public List<PartitionSpec> filterPartitionSpecs(List<PartitionSpec> partitionSpecList) + List<PartitionSpec> filterPartitionSpecs(List<PartitionSpec> partitionSpecList) throws MetaException; /** @@ -97,18 +131,17 @@ public interface MetaStoreFilterHook { * @return the same partition object if it's not filtered out * @throws NoSuchObjectException */ - public Partition filterPartition(Partition partition) throws MetaException, NoSuchObjectException; + Partition filterPartition(Partition partition) throws MetaException, NoSuchObjectException; /** * Filter given list of partition names - * @param dbName - * @param tblName - * @param partitionNames - * @return + * @param catName catalog name. + * @param dbName database name. + * @param tblName table name. + * @param partitionNames list of partition names. + * @return list of filtered partition names. */ - public List<String> filterPartitionNames(String dbName, String tblName, + List<String> filterPartitionNames(String catName, String dbName, String tblName, List<String> partitionNames) throws MetaException; - - } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java index f5a91b4..988fca6 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreListenerNotifier.java @@ -35,10 +35,12 @@ import org.apache.hadoop.hive.metastore.events.AlterISchemaEvent; import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; import org.apache.hadoop.hive.metastore.events.AlterSchemaVersionEvent; import org.apache.hadoop.hive.metastore.events.AlterTableEvent; +import org.apache.hadoop.hive.metastore.events.CreateCatalogEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; import org.apache.hadoop.hive.metastore.events.CreateFunctionEvent; import org.apache.hadoop.hive.metastore.events.CreateISchemaEvent; import org.apache.hadoop.hive.metastore.events.CreateTableEvent; +import org.apache.hadoop.hive.metastore.events.DropCatalogEvent; import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropFunctionEvent; import org.apache.hadoop.hive.metastore.events.DropISchemaEvent; @@ -200,6 +202,10 @@ public class MetaStoreListenerNotifier { listener.onDropSchemaVersion((DropSchemaVersionEvent) event); } }) + .put(EventType.CREATE_CATALOG, + (listener, event) -> listener.onCreateCatalog((CreateCatalogEvent)event)) + .put(EventType.DROP_CATALOG, + (listener, event) -> listener.onDropCatalog((DropCatalogEvent)event)) .build() );
