http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java index cf92eda..89b4006 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/SharedCache.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.metastore.cache; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -30,11 +31,15 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.TreeMap; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.StatObjectConverter; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -50,29 +55,38 @@ import org.slf4j.LoggerFactory; import com.google.common.annotations.VisibleForTesting; +import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; + public class SharedCache { private static ReentrantReadWriteLock cacheLock = new ReentrantReadWriteLock(true); + private boolean isCatalogCachePrewarmed = false; + private Map<String, Catalog> catalogCache = new TreeMap<>(); + private HashSet<String> catalogsDeletedDuringPrewarm = new HashSet<>(); + private AtomicBoolean isCatalogCacheDirty = new AtomicBoolean(false); + // For caching Database objects. Key is database name - private Map<String, Database> databaseCache = new ConcurrentHashMap<String, Database>(); + private Map<String, Database> databaseCache = new TreeMap<>(); private boolean isDatabaseCachePrewarmed = false; - private HashSet<String> databasesDeletedDuringPrewarm = new HashSet<String>(); + private HashSet<String> databasesDeletedDuringPrewarm = new HashSet<>(); private AtomicBoolean isDatabaseCacheDirty = new AtomicBoolean(false); + // For caching TableWrapper objects. Key is aggregate of database name and table name - private Map<String, TableWrapper> tableCache = new ConcurrentHashMap<String, TableWrapper>(); + private Map<String, TableWrapper> tableCache = new TreeMap<>(); private boolean isTableCachePrewarmed = false; - private HashSet<String> tablesDeletedDuringPrewarm = new HashSet<String>(); + private HashSet<String> tablesDeletedDuringPrewarm = new HashSet<>(); private AtomicBoolean isTableCacheDirty = new AtomicBoolean(false); - private Map<ByteArrayWrapper, StorageDescriptorWrapper> sdCache = new ConcurrentHashMap<>(); + private Map<ByteArrayWrapper, StorageDescriptorWrapper> sdCache = new HashMap<>(); + private Configuration conf; private static MessageDigest md; static final private Logger LOG = LoggerFactory.getLogger(SharedCache.class.getName()); private AtomicLong cacheUpdateCount = new AtomicLong(0); - static enum StatsType { + enum StatsType { ALL(0), ALLBUTDEFAULT(1); private final int position; - private StatsType(int position) { + StatsType(int position) { this.position = position; } @@ -155,6 +169,10 @@ public class SharedCache { this.parameters = parameters; } + boolean sameDatabase(String catName, String dbName) { + return catName.equals(t.getCatName()) && dbName.equals(t.getDbName()); + } + void cachePartition(Partition part, SharedCache sharedCache) { try { tableLock.writeLock().lock(); @@ -669,12 +687,102 @@ public class SharedCache { } } - public Database getDatabaseFromCache(String name) { + public void populateCatalogsInCache(Collection<Catalog> catalogs) { + for (Catalog cat : catalogs) { + Catalog catCopy = cat.deepCopy(); + // ObjectStore also stores db name in lowercase + catCopy.setName(catCopy.getName().toLowerCase()); + try { + cacheLock.writeLock().lock(); + // Since we allow write operations on cache while prewarm is happening: + // 1. Don't add databases that were deleted while we were preparing list for prewarm + // 2. Skip overwriting exisiting db object + // (which is present because it was added after prewarm started) + if (catalogsDeletedDuringPrewarm.contains(catCopy.getName())) { + continue; + } + catalogCache.putIfAbsent(catCopy.getName(), catCopy); + catalogsDeletedDuringPrewarm.clear(); + isCatalogCachePrewarmed = true; + } finally { + cacheLock.writeLock().unlock(); + } + } + } + + public Catalog getCatalogFromCache(String name) { + Catalog cat = null; + try { + cacheLock.readLock().lock(); + if (catalogCache.get(name) != null) { + cat = catalogCache.get(name).deepCopy(); + } + } finally { + cacheLock.readLock().unlock(); + } + return cat; + } + + public void addCatalogToCache(Catalog cat) { + try { + cacheLock.writeLock().lock(); + Catalog catCopy = cat.deepCopy(); + // ObjectStore also stores db name in lowercase + catCopy.setName(catCopy.getName().toLowerCase()); + catalogCache.put(cat.getName(), catCopy); + isCatalogCacheDirty.set(true); + } finally { + cacheLock.writeLock().unlock(); + } + } + + public void alterCatalogInCache(String catName, Catalog newCat) { + try { + cacheLock.writeLock().lock(); + removeCatalogFromCache(catName); + addCatalogToCache(newCat.deepCopy()); + } finally { + cacheLock.writeLock().unlock(); + } + } + + public void removeCatalogFromCache(String name) { + name = normalizeIdentifier(name); + try { + cacheLock.writeLock().lock(); + // If db cache is not yet prewarmed, add this to a set which the prewarm thread can check + // so that the prewarm thread does not add it back + if (!isCatalogCachePrewarmed) { + catalogsDeletedDuringPrewarm.add(name); + } + if (catalogCache.remove(name) != null) { + isCatalogCacheDirty.set(true); + } + } finally { + cacheLock.writeLock().unlock(); + } + } + + public List<String> listCachedCatalogs() { + try { + cacheLock.readLock().lock(); + return new ArrayList<>(catalogCache.keySet()); + } finally { + cacheLock.readLock().unlock(); + } + } + + public boolean isCatalogCachePrewarmed() { + return isCatalogCachePrewarmed; + } + + public Database getDatabaseFromCache(String catName, String name) { Database db = null; try { cacheLock.readLock().lock(); - if (databaseCache.get(name) != null) { - db = databaseCache.get(name).deepCopy(); + String key = CacheUtils.buildDbKey(catName, name); + if (databaseCache.get(key) != null) { + db = databaseCache.get(key).deepCopy(); } } finally { cacheLock.readLock().unlock(); @@ -693,10 +801,11 @@ public class SharedCache { // 1. Don't add databases that were deleted while we were preparing list for prewarm // 2. Skip overwriting exisiting db object // (which is present because it was added after prewarm started) - if (databasesDeletedDuringPrewarm.contains(dbCopy.getName().toLowerCase())) { + String key = CacheUtils.buildDbKey(dbCopy.getCatalogName().toLowerCase(), dbCopy.getName().toLowerCase()); + if (databasesDeletedDuringPrewarm.contains(key)) { continue; } - databaseCache.putIfAbsent(StringUtils.normalizeIdentifier(dbCopy.getName()), dbCopy); + databaseCache.putIfAbsent(key, dbCopy); databasesDeletedDuringPrewarm.clear(); isDatabaseCachePrewarmed = true; } finally { @@ -715,22 +824,24 @@ public class SharedCache { Database dbCopy = db.deepCopy(); // ObjectStore also stores db name in lowercase dbCopy.setName(dbCopy.getName().toLowerCase()); - databaseCache.put(StringUtils.normalizeIdentifier(dbCopy.getName()), dbCopy); + dbCopy.setCatalogName(dbCopy.getCatalogName().toLowerCase()); + databaseCache.put(CacheUtils.buildDbKey(dbCopy.getCatalogName(), dbCopy.getName()), dbCopy); isDatabaseCacheDirty.set(true); } finally { cacheLock.writeLock().unlock(); } } - public void removeDatabaseFromCache(String dbName) { + public void removeDatabaseFromCache(String catName, String dbName) { try { cacheLock.writeLock().lock(); // If db cache is not yet prewarmed, add this to a set which the prewarm thread can check // so that the prewarm thread does not add it back + String key = CacheUtils.buildDbKey(catName, dbName); if (!isDatabaseCachePrewarmed) { - databasesDeletedDuringPrewarm.add(dbName.toLowerCase()); + databasesDeletedDuringPrewarm.add(key); } - if (databaseCache.remove(dbName) != null) { + if (databaseCache.remove(key) != null) { isDatabaseCacheDirty.set(true); } } finally { @@ -738,25 +849,31 @@ public class SharedCache { } } - public List<String> listCachedDatabases() { + public List<String> listCachedDatabases(String catName) { List<String> results = new ArrayList<>(); try { cacheLock.readLock().lock(); - results.addAll(databaseCache.keySet()); + for (String pair : databaseCache.keySet()) { + String[] n = CacheUtils.splitDbName(pair); + if (catName.equals(n[0])) results.add(n[1]); + } } finally { cacheLock.readLock().unlock(); } return results; } - public List<String> listCachedDatabases(String pattern) { + public List<String> listCachedDatabases(String catName, String pattern) { List<String> results = new ArrayList<>(); try { cacheLock.readLock().lock(); - for (String dbName : databaseCache.keySet()) { - dbName = StringUtils.normalizeIdentifier(dbName); - if (CacheUtils.matches(dbName, pattern)) { - results.add(dbName); + for (String pair : databaseCache.keySet()) { + String[] n = CacheUtils.splitDbName(pair); + if (catName.equals(n[0])) { + n[1] = StringUtils.normalizeIdentifier(n[1]); + if (CacheUtils.matches(n[1], pattern)) { + results.add(n[1]); + } } } } finally { @@ -768,13 +885,11 @@ public class SharedCache { /** * Replaces the old db object with the new one. * This will add the new database to cache if it does not exist. - * @param dbName - * @param newDb */ - public void alterDatabaseInCache(String dbName, Database newDb) { + public void alterDatabaseInCache(String catName, String dbName, Database newDb) { try { cacheLock.writeLock().lock(); - removeDatabaseFromCache(dbName); + removeDatabaseFromCache(catName, dbName); addDatabaseToCache(newDb.deepCopy()); isDatabaseCacheDirty.set(true); } finally { @@ -810,14 +925,15 @@ public class SharedCache { public void populateTableInCache(Table table, ColumnStatistics tableColStats, List<Partition> partitions, List<ColumnStatistics> partitionColStats, AggrStats aggrStatsAllPartitions, AggrStats aggrStatsAllButDefaultPartition) { + String catName = StringUtils.normalizeIdentifier(table.getCatName()); String dbName = StringUtils.normalizeIdentifier(table.getDbName()); String tableName = StringUtils.normalizeIdentifier(table.getTableName()); // Since we allow write operations on cache while prewarm is happening: // 1. Don't add tables that were deleted while we were preparing list for prewarm - if (tablesDeletedDuringPrewarm.contains(CacheUtils.buildTableCacheKey(dbName, tableName))) { + if (tablesDeletedDuringPrewarm.contains(CacheUtils.buildTableKey(catName, dbName, tableName))) { return; } - TableWrapper tblWrapper = createTableWrapper(dbName, tableName, table); + TableWrapper tblWrapper = createTableWrapper(catName, dbName, tableName, table); if (!table.isSetPartitionKeys() && (tableColStats != null)) { tblWrapper.updateTableColStats(tableColStats.getStatsObj()); } else { @@ -843,12 +959,14 @@ public class SharedCache { cacheLock.writeLock().lock(); // 2. Skip overwriting exisiting table object // (which is present because it was added after prewarm started) - tableCache.putIfAbsent(CacheUtils.buildTableCacheKey(dbName, tableName), tblWrapper); + tableCache.putIfAbsent(CacheUtils.buildTableKey(catName, dbName, tableName), tblWrapper); } finally { cacheLock.writeLock().unlock(); } } + + public void completeTableCachePrewarm() { try { cacheLock.writeLock().lock(); @@ -859,11 +977,11 @@ public class SharedCache { } } - public Table getTableFromCache(String dbName, String tableName) { + public Table getTableFromCache(String catName, String dbName, String tableName) { Table t = null; try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tableName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tableName)); if (tblWrapper != null) { t = CacheUtils.assemble(tblWrapper, this); } @@ -873,11 +991,11 @@ public class SharedCache { return t; } - public TableWrapper addTableToCache(String dbName, String tblName, Table tbl) { + public TableWrapper addTableToCache(String catName, String dbName, String tblName, Table tbl) { try { cacheLock.writeLock().lock(); - TableWrapper wrapper = createTableWrapper(dbName, tblName, tbl); - tableCache.put(CacheUtils.buildTableCacheKey(dbName, tblName), wrapper); + TableWrapper wrapper = createTableWrapper(catName, dbName, tblName, tbl); + tableCache.put(CacheUtils.buildTableKey(catName, dbName, tblName), wrapper); isTableCacheDirty.set(true); return wrapper; } finally { @@ -885,14 +1003,15 @@ public class SharedCache { } } - private TableWrapper createTableWrapper(String dbName, String tblName, Table tbl) { + private TableWrapper createTableWrapper(String catName, String dbName, String tblName, Table tbl) { TableWrapper wrapper; Table tblCopy = tbl.deepCopy(); - tblCopy.setDbName(StringUtils.normalizeIdentifier(dbName)); - tblCopy.setTableName(StringUtils.normalizeIdentifier(tblName)); + tblCopy.setCatName(normalizeIdentifier(catName)); + tblCopy.setDbName(normalizeIdentifier(dbName)); + tblCopy.setTableName(normalizeIdentifier(tblName)); if (tblCopy.getPartitionKeys() != null) { for (FieldSchema fs : tblCopy.getPartitionKeys()) { - fs.setName(StringUtils.normalizeIdentifier(fs.getName())); + fs.setName(normalizeIdentifier(fs.getName())); } } if (tbl.getSd() != null) { @@ -907,15 +1026,16 @@ public class SharedCache { return wrapper; } - public void removeTableFromCache(String dbName, String tblName) { + + public void removeTableFromCache(String catName, String dbName, String tblName) { try { cacheLock.writeLock().lock(); // If table cache is not yet prewarmed, add this to a set which the prewarm thread can check // so that the prewarm thread does not add it back if (!isTableCachePrewarmed) { - tablesDeletedDuringPrewarm.add(CacheUtils.buildTableCacheKey(dbName, tblName)); + tablesDeletedDuringPrewarm.add(CacheUtils.buildTableKey(catName, dbName, tblName)); } - TableWrapper tblWrapper = tableCache.remove(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.remove(CacheUtils.buildTableKey(catName, dbName, tblName)); byte[] sdHash = tblWrapper.getSdHash(); if (sdHash != null) { decrSd(sdHash); @@ -926,15 +1046,15 @@ public class SharedCache { } } - public void alterTableInCache(String dbName, String tblName, Table newTable) { + public void alterTableInCache(String catName, String dbName, String tblName, Table newTable) { try { cacheLock.writeLock().lock(); - TableWrapper tblWrapper = tableCache.remove(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.remove(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.updateTableObj(newTable, this); String newDbName = StringUtils.normalizeIdentifier(newTable.getDbName()); String newTblName = StringUtils.normalizeIdentifier(newTable.getTableName()); - tableCache.put(CacheUtils.buildTableCacheKey(newDbName, newTblName), tblWrapper); + tableCache.put(CacheUtils.buildTableKey(catName, newDbName, newTblName), tblWrapper); isTableCacheDirty.set(true); } } finally { @@ -942,12 +1062,12 @@ public class SharedCache { } } - public List<Table> listCachedTables(String dbName) { + public List<Table> listCachedTables(String catName, String dbName) { List<Table> tables = new ArrayList<>(); try { cacheLock.readLock().lock(); for (TableWrapper wrapper : tableCache.values()) { - if (wrapper.getTable().getDbName().equals(dbName)) { + if (wrapper.sameDatabase(catName, dbName)) { tables.add(CacheUtils.assemble(wrapper, this)); } } @@ -957,12 +1077,12 @@ public class SharedCache { return tables; } - public List<String> listCachedTableNames(String dbName) { + public List<String> listCachedTableNames(String catName, String dbName) { List<String> tableNames = new ArrayList<>(); try { cacheLock.readLock().lock(); for (TableWrapper wrapper : tableCache.values()) { - if (wrapper.getTable().getDbName().equals(dbName)) { + if (wrapper.sameDatabase(catName, dbName)) { tableNames.add(StringUtils.normalizeIdentifier(wrapper.getTable().getTableName())); } } @@ -972,13 +1092,13 @@ public class SharedCache { return tableNames; } - public List<String> listCachedTableNames(String dbName, String pattern, short maxTables) { - List<String> tableNames = new ArrayList<String>(); + public List<String> listCachedTableNames(String catName, String dbName, String pattern, short maxTables) { + List<String> tableNames = new ArrayList<>(); try { cacheLock.readLock().lock(); int count = 0; for (TableWrapper wrapper : tableCache.values()) { - if ((wrapper.getTable().getDbName().equals(dbName)) + if (wrapper.sameDatabase(catName, dbName) && CacheUtils.matches(wrapper.getTable().getTableName(), pattern) && (maxTables == -1 || count < maxTables)) { tableNames.add(StringUtils.normalizeIdentifier(wrapper.getTable().getTableName())); @@ -991,12 +1111,12 @@ public class SharedCache { return tableNames; } - public List<String> listCachedTableNames(String dbName, String pattern, TableType tableType) { - List<String> tableNames = new ArrayList<String>(); + public List<String> listCachedTableNames(String catName, String dbName, String pattern, TableType tableType) { + List<String> tableNames = new ArrayList<>(); try { cacheLock.readLock().lock(); for (TableWrapper wrapper : tableCache.values()) { - if ((wrapper.getTable().getDbName().equals(dbName)) + if (wrapper.sameDatabase(catName, dbName) && CacheUtils.matches(wrapper.getTable().getTableName(), pattern) && wrapper.getTable().getTableType().equals(tableType.toString())) { tableNames.add(StringUtils.normalizeIdentifier(wrapper.getTable().getTableName())); @@ -1008,23 +1128,23 @@ public class SharedCache { return tableNames; } - public void refreshTablesInCache(String dbName, List<Table> tables) { + public void refreshTablesInCache(String catName, String dbName, List<Table> tables) { try { cacheLock.writeLock().lock(); if (isTableCacheDirty.compareAndSet(true, false)) { LOG.debug("Skipping table cache update; the table list we have is dirty."); return; } - Map<String, TableWrapper> newTableCache = new HashMap<String, TableWrapper>(); + Map<String, TableWrapper> newTableCache = new HashMap<>(); for (Table tbl : tables) { String tblName = StringUtils.normalizeIdentifier(tbl.getTableName()); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.updateTableObj(tbl, this); } else { - tblWrapper = createTableWrapper(dbName, tblName, tbl); + tblWrapper = createTableWrapper(catName, dbName, tblName, tbl); } - newTableCache.put(CacheUtils.buildTableCacheKey(dbName, tblName), tblWrapper); + newTableCache.put(CacheUtils.buildTableKey(catName, dbName, tblName), tblWrapper); } tableCache.clear(); tableCache = newTableCache; @@ -1033,12 +1153,12 @@ public class SharedCache { } } - public List<ColumnStatisticsObj> getTableColStatsFromCache(String dbName, String tblName, - List<String> colNames) { - List<ColumnStatisticsObj> colStatObjs = new ArrayList<ColumnStatisticsObj>(); + public List<ColumnStatisticsObj> getTableColStatsFromCache( + String catName, String dbName, String tblName, List<String> colNames) { + List<ColumnStatisticsObj> colStatObjs = new ArrayList<>(); try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { colStatObjs = tblWrapper.getCachedTableColStats(colNames); } @@ -1048,10 +1168,10 @@ public class SharedCache { return colStatObjs; } - public void removeTableColStatsFromCache(String dbName, String tblName, String colName) { + public void removeTableColStatsFromCache(String catName, String dbName, String tblName, String colName) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.removeTableColStats(colName); } @@ -1060,11 +1180,11 @@ public class SharedCache { } } - public void updateTableColStatsInCache(String dbName, String tableName, - List<ColumnStatisticsObj> colStatsForTable) { + public void updateTableColStatsInCache(String catName, String dbName, String tableName, + List<ColumnStatisticsObj> colStatsForTable) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tableName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tableName)); if (tblWrapper != null) { tblWrapper.updateTableColStats(colStatsForTable); } @@ -1073,11 +1193,11 @@ public class SharedCache { } } - public void refreshTableColStatsInCache(String dbName, String tableName, - List<ColumnStatisticsObj> colStatsForTable) { + public void refreshTableColStatsInCache(String catName, String dbName, String tableName, + List<ColumnStatisticsObj> colStatsForTable) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tableName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tableName)); if (tblWrapper != null) { tblWrapper.refreshTableColStats(colStatsForTable); } @@ -1095,18 +1215,19 @@ public class SharedCache { } } - public List<TableMeta> getTableMeta(String dbNames, String tableNames, - List<String> tableTypes) { + public List<TableMeta> getTableMeta(String catName, String dbNames, String tableNames, + List<String> tableTypes) { List<TableMeta> tableMetas = new ArrayList<>(); try { cacheLock.readLock().lock(); - for (String dbName : listCachedDatabases()) { + for (String dbName : listCachedDatabases(catName)) { if (CacheUtils.matches(dbName, dbNames)) { - for (Table table : listCachedTables(dbName)) { + for (Table table : listCachedTables(catName, dbName)) { if (CacheUtils.matches(table.getTableName(), tableNames)) { if (tableTypes == null || tableTypes.contains(table.getTableType())) { TableMeta metaData = new TableMeta(dbName, table.getTableName(), table.getTableType()); + metaData.setCatName(catName); metaData.setComments(table.getParameters().get("comment")); tableMetas.add(metaData); } @@ -1120,10 +1241,10 @@ public class SharedCache { return tableMetas; } - public void addPartitionToCache(String dbName, String tblName, Partition part) { + public void addPartitionToCache(String catName, String dbName, String tblName, Partition part) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.cachePartition(part, this); } @@ -1132,10 +1253,10 @@ public class SharedCache { } } - public void addPartitionsToCache(String dbName, String tblName, List<Partition> parts) { + public void addPartitionsToCache(String catName, String dbName, String tblName, List<Partition> parts) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.cachePartitions(parts, this); } @@ -1144,12 +1265,12 @@ public class SharedCache { } } - public Partition getPartitionFromCache(String dbName, String tblName, - List<String> partVals) { + public Partition getPartitionFromCache(String catName, String dbName, String tblName, + List<String> partVals) { Partition part = null; try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { part = tblWrapper.getPartition(partVals, this); } @@ -1159,11 +1280,11 @@ public class SharedCache { return part; } - public boolean existPartitionFromCache(String dbName, String tblName, List<String> partVals) { + public boolean existPartitionFromCache(String catName, String dbName, String tblName, List<String> partVals) { boolean existsPart = false; try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { existsPart = tblWrapper.containsPartition(partVals); } @@ -1173,12 +1294,12 @@ public class SharedCache { return existsPart; } - public Partition removePartitionFromCache(String dbName, String tblName, - List<String> partVals) { + public Partition removePartitionFromCache(String catName, String dbName, String tblName, + List<String> partVals) { Partition part = null; try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { part = tblWrapper.removePartition(partVals, this); } @@ -1188,11 +1309,11 @@ public class SharedCache { return part; } - public void removePartitionsFromCache(String dbName, String tblName, - List<List<String>> partVals) { + public void removePartitionsFromCache(String catName, String dbName, String tblName, + List<List<String>> partVals) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.removePartitions(partVals, this); } @@ -1201,11 +1322,11 @@ public class SharedCache { } } - public List<Partition> listCachedPartitions(String dbName, String tblName, int max) { + public List<Partition> listCachedPartitions(String catName, String dbName, String tblName, int max) { List<Partition> parts = new ArrayList<Partition>(); try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { parts = tblWrapper.listPartitions(max, this); } @@ -1215,11 +1336,11 @@ public class SharedCache { return parts; } - public void alterPartitionInCache(String dbName, String tblName, List<String> partVals, - Partition newPart) { + public void alterPartitionInCache(String catName, String dbName, String tblName, List<String> partVals, + Partition newPart) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.alterPartition(partVals, newPart, this); } @@ -1228,11 +1349,11 @@ public class SharedCache { } } - public void alterPartitionsInCache(String dbName, String tblName, List<List<String>> partValsList, - List<Partition> newParts) { + public void alterPartitionsInCache(String catName, String dbName, String tblName, List<List<String>> partValsList, + List<Partition> newParts) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.alterPartitions(partValsList, newParts, this); } @@ -1241,10 +1362,10 @@ public class SharedCache { } } - public void refreshPartitionsInCache(String dbName, String tblName, List<Partition> partitions) { + public void refreshPartitionsInCache(String catName, String dbName, String tblName, List<Partition> partitions) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.refreshPartitions(partitions, this); } @@ -1253,11 +1374,11 @@ public class SharedCache { } } - public void removePartitionColStatsFromCache(String dbName, String tblName, - List<String> partVals, String colName) { + public void removePartitionColStatsFromCache(String catName, String dbName, String tblName, + List<String> partVals, String colName) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.removePartitionColStats(partVals, colName); } @@ -1266,11 +1387,11 @@ public class SharedCache { } } - public void updatePartitionColStatsInCache(String dbName, String tableName, - List<String> partVals, List<ColumnStatisticsObj> colStatsObjs) { + public void updatePartitionColStatsInCache(String catName, String dbName, String tableName, + List<String> partVals, List<ColumnStatisticsObj> colStatsObjs) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tableName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tableName)); if (tblWrapper != null) { tblWrapper.updatePartitionColStats(partVals, colStatsObjs); } @@ -1279,12 +1400,12 @@ public class SharedCache { } } - public ColumnStatisticsObj getPartitionColStatsFromCache(String dbName, String tblName, - List<String> partVal, String colName) { + public ColumnStatisticsObj getPartitionColStatsFromCache(String catName, String dbName, String tblName, + List<String> partVal, String colName) { ColumnStatisticsObj colStatObj = null; try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null){ colStatObj = tblWrapper.getPartitionColStats(partVal, colName); } @@ -1294,11 +1415,11 @@ public class SharedCache { return colStatObj; } - public void refreshPartitionColStatsInCache(String dbName, String tblName, - List<ColumnStatistics> partitionColStats) { + public void refreshPartitionColStatsInCache(String catName, String dbName, String tblName, + List<ColumnStatistics> partitionColStats) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.refreshPartitionColStats(partitionColStats); } @@ -1307,11 +1428,11 @@ public class SharedCache { } } - public List<ColumnStatisticsObj> getAggrStatsFromCache(String dbName, String tblName, - List<String> colNames, StatsType statsType) { + public List<ColumnStatisticsObj> getAggrStatsFromCache(String catName, String dbName, String tblName, + List<String> colNames, StatsType statsType) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { return tblWrapper.getAggrPartitionColStats(colNames, statsType); } @@ -1321,11 +1442,11 @@ public class SharedCache { return null; } - public void addAggregateStatsToCache(String dbName, String tblName, - AggrStats aggrStatsAllPartitions, AggrStats aggrStatsAllButDefaultPartition) { + public void addAggregateStatsToCache(String catName, String dbName, String tblName, + AggrStats aggrStatsAllPartitions, AggrStats aggrStatsAllButDefaultPartition) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null){ tblWrapper.cacheAggrPartitionColStats(aggrStatsAllPartitions, aggrStatsAllButDefaultPartition); @@ -1335,11 +1456,11 @@ public class SharedCache { } } - public void refreshAggregateStatsInCache(String dbName, String tblName, - AggrStats aggrStatsAllPartitions, AggrStats aggrStatsAllButDefaultPartition) { + public void refreshAggregateStatsInCache(String catName, String dbName, String tblName, + AggrStats aggrStatsAllPartitions, AggrStats aggrStatsAllButDefaultPartition) { try { cacheLock.readLock().lock(); - TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableCacheKey(dbName, tblName)); + TableWrapper tblWrapper = tableCache.get(CacheUtils.buildTableKey(catName, dbName, tblName)); if (tblWrapper != null) { tblWrapper.refreshAggrPartitionColStats(aggrStatsAllPartitions, aggrStatsAllButDefaultPartition); @@ -1390,6 +1511,16 @@ public class SharedCache { return sdCache; } + /** + * This resets the contents of the cataog cache so that we can re-fill it in another test. + */ + void resetCatalogCache() { + isCatalogCachePrewarmed = false; + catalogCache.clear(); + catalogsDeletedDuringPrewarm.clear(); + isCatalogCacheDirty.set(false); + } + public long getUpdateCount() { return cacheUpdateCount.get(); } @@ -1398,3 +1529,8 @@ public class SharedCache { cacheUpdateCount.incrementAndGet(); } } + + + + +
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/CatalogBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/CatalogBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/CatalogBuilder.java new file mode 100644 index 0000000..be76d93 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/CatalogBuilder.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.Catalog; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.thrift.TException; + +public class CatalogBuilder { + private String name, description, location; + + public CatalogBuilder setName(String name) { + this.name = name; + return this; + } + + public CatalogBuilder setDescription(String description) { + this.description = description; + return this; + } + + public CatalogBuilder setLocation(String location) { + this.location = location; + return this; + } + + public Catalog build() throws MetaException { + if (name == null) throw new MetaException("You must name the catalog"); + if (location == null) throw new MetaException("You must give the catalog a location"); + Catalog catalog = new Catalog(name, location); + if (description != null) catalog.setDescription(description); + return catalog; + } + + /** + * Build the catalog object and create it in the metastore. + * @param client metastore client + * @return new catalog object + * @throws TException thrown from the client + */ + public Catalog create(IMetaStoreClient client) throws TException { + Catalog cat = build(); + client.createCatalog(cat); + return cat; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ConstraintBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ConstraintBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ConstraintBuilder.java index 50e779a..2e32cbf 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ConstraintBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ConstraintBuilder.java @@ -17,8 +17,15 @@ */ package org.apache.hadoop.hive.metastore.client.builder; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; + +import java.util.ArrayList; +import java.util.List; /** * Base builder for all types of constraints. Database name, table name, and column name @@ -26,28 +33,42 @@ import org.apache.hadoop.hive.metastore.api.Table; * @param <T> Type of builder extending this. */ abstract class ConstraintBuilder<T> { - protected String dbName, tableName, columnName, constraintName; - protected int keySeq; + protected String catName, dbName, tableName, constraintName; + List<String> columns; protected boolean enable, validate, rely; + private int nextSeq; private T child; protected ConstraintBuilder() { - keySeq = 1; + nextSeq = 1; enable = true; validate = rely = false; + dbName = Warehouse.DEFAULT_DATABASE_NAME; + columns = new ArrayList<>(); } protected void setChild(T child) { this.child = child; } - protected void checkBuildable(String defaultConstraintName) throws MetaException { - if (dbName == null || tableName == null || columnName == null) { - throw new MetaException("You must provide database name, table name, and column name"); + protected void checkBuildable(String defaultConstraintName, Configuration conf) + throws MetaException { + if (tableName == null || columns.isEmpty()) { + throw new MetaException("You must provide table name and columns"); } if (constraintName == null) { - constraintName = dbName + "_" + tableName + "_" + columnName + "_" + defaultConstraintName; + constraintName = tableName + "_" + defaultConstraintName; } + if (catName == null) catName = MetaStoreUtils.getDefaultCatalog(conf); + } + + protected int getNextSeq() { + return nextSeq++; + } + + public T setCatName(String catName) { + this.catName = catName; + return child; } public T setDbName(String dbName) { @@ -60,14 +81,15 @@ abstract class ConstraintBuilder<T> { return child; } - public T setDbAndTableName(Table table) { + public T onTable(Table table) { + this.catName = table.getCatName(); this.dbName = table.getDbName(); this.tableName = table.getTableName(); return child; } - public T setColumnName(String columnName) { - this.columnName = columnName; + public T addColumn(String columnName) { + this.columns.add(columnName); return child; } @@ -76,11 +98,6 @@ abstract class ConstraintBuilder<T> { return child; } - public T setKeySeq(int keySeq) { - this.keySeq = keySeq; - return child; - } - public T setEnable(boolean enable) { this.enable = enable; return child; http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java index 01693ec..f3d2182 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/DatabaseBuilder.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hive.metastore.client.builder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.PrincipalType; @@ -33,11 +37,24 @@ import java.util.Map; * selects reasonable defaults. */ public class DatabaseBuilder { - private String name, description, location; + private String name, description, location, catalogName; private Map<String, String> params = new HashMap<>(); private String ownerName; private PrincipalType ownerType; + public DatabaseBuilder() { + } + + public DatabaseBuilder setCatalogName(String catalogName) { + this.catalogName = catalogName; + return this; + } + + public DatabaseBuilder setCatalogName(Catalog catalog) { + this.catalogName = catalog.getName(); + return this; + } + public DatabaseBuilder setName(String name) { this.name = name; return this; @@ -73,11 +90,13 @@ public class DatabaseBuilder { return this; } - public Database build() throws MetaException { + public Database build(Configuration conf) throws MetaException { if (name == null) throw new MetaException("You must name the database"); + if (catalogName == null) catalogName = MetaStoreUtils.getDefaultCatalog(conf); Database db = new Database(name, description, location, params); + db.setCatalogName(catalogName); try { - if (ownerName != null) ownerName = SecurityUtils.getUser(); + if (ownerName == null) ownerName = SecurityUtils.getUser(); db.setOwnerName(ownerName); if (ownerType == null) ownerType = PrincipalType.USER; db.setOwnerType(ownerType); @@ -86,4 +105,18 @@ public class DatabaseBuilder { throw MetaStoreUtils.newMetaException(e); } } + + /** + * Build the database, create it in the metastore, and then return the db object. + * @param client metastore client + * @param conf configuration file + * @return new database object + * @throws TException comes from {@link #build(Configuration)} or + * {@link IMetaStoreClient#createDatabase(Database)}. + */ + public Database create(IMetaStoreClient client, Configuration conf) throws TException { + Database db = build(conf); + client.createDatabase(db); + return db; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/FunctionBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/FunctionBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/FunctionBuilder.java index aa9b9f5..c4c09dc 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/FunctionBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/FunctionBuilder.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.metastore.client.builder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.FunctionType; @@ -26,6 +29,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.ResourceUri; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; +import org.apache.thrift.TException; import java.io.IOException; import java.util.ArrayList; @@ -35,7 +39,7 @@ import java.util.List; * Class for creating Thrift Function objects for tests, and API usage. */ public class FunctionBuilder { - private String dbName = "default"; + private String catName, dbName; private String funcName = null; private String className = null; private String owner = null; @@ -49,7 +53,13 @@ public class FunctionBuilder { ownerType = PrincipalType.USER; createTime = (int) (System.currentTimeMillis() / 1000); funcType = FunctionType.JAVA; - resourceUris = new ArrayList<ResourceUri>(); + resourceUris = new ArrayList<>(); + dbName = Warehouse.DEFAULT_DATABASE_NAME; + } + + public FunctionBuilder setCatName(String catName) { + this.catName = catName; + return this; } public FunctionBuilder setDbName(String dbName) { @@ -57,8 +67,9 @@ public class FunctionBuilder { return this; } - public FunctionBuilder setDbName(Database db) { + public FunctionBuilder inDb(Database db) { this.dbName = db.getName(); + this.catName = db.getCatalogName(); return this; } @@ -102,7 +113,7 @@ public class FunctionBuilder { return this; } - public Function build() throws MetaException { + public Function build(Configuration conf) throws MetaException { try { if (owner != null) { owner = SecurityUtils.getUser(); @@ -110,7 +121,23 @@ public class FunctionBuilder { } catch (IOException e) { throw MetaStoreUtils.newMetaException(e); } - return new Function(funcName, dbName, className, owner, ownerType, createTime, funcType, + if (catName == null) catName = MetaStoreUtils.getDefaultCatalog(conf); + Function f = new Function(funcName, dbName, className, owner, ownerType, createTime, funcType, resourceUris); + f.setCatName(catName); + return f; + } + + /** + * Create the function object in the metastore and return it. + * @param client metastore client + * @param conf configuration + * @return new function object + * @throws TException if thrown by build or the client. + */ + public Function create(IMetaStoreClient client, Configuration conf) throws TException { + Function f = build(conf); + client.createFunction(f); + return f; } } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ISchemaBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ISchemaBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ISchemaBuilder.java index 32a84ac..f61a62c 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ISchemaBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/ISchemaBuilder.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.metastore.client.builder; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.ISchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.SchemaCompatibility; @@ -27,7 +28,7 @@ import org.apache.hadoop.hive.metastore.api.SchemaValidation; public class ISchemaBuilder { private SchemaType schemaType; // required private String name; // required - private String dbName; // required + private String dbName, catName; // required private SchemaCompatibility compatibility; // required private SchemaValidation validationLevel; // required private boolean canEvolve; // required @@ -39,6 +40,7 @@ public class ISchemaBuilder { validationLevel = SchemaValidation.ALL; canEvolve = true; dbName = Warehouse.DEFAULT_DATABASE_NAME; + catName = Warehouse.DEFAULT_CATALOG_NAME; } public ISchemaBuilder setSchemaType(SchemaType schemaType) { @@ -56,6 +58,12 @@ public class ISchemaBuilder { return this; } + public ISchemaBuilder inDb(Database db) { + this.catName = db.getCatalogName(); + this.dbName = db.getName(); + return this; + } + public ISchemaBuilder setCompatibility(SchemaCompatibility compatibility) { this.compatibility = compatibility; return this; @@ -86,7 +94,7 @@ public class ISchemaBuilder { throw new MetaException("You must provide a schemaType and name"); } ISchema iSchema = - new ISchema(schemaType, name, dbName, compatibility, validationLevel, canEvolve); + new ISchema(schemaType, name, catName, dbName, compatibility, validationLevel, canEvolve); if (schemaGroup != null) iSchema.setSchemaGroup(schemaGroup); if (description != null) iSchema.setDescription(description); return iSchema; http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java index 38e5a8f..d6ee673 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/PartitionBuilder.java @@ -17,9 +17,14 @@ */ package org.apache.hadoop.hive.metastore.client.builder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.thrift.TException; import java.util.ArrayList; import java.util.HashMap; @@ -31,7 +36,7 @@ import java.util.Map; * reference; 2. partition values; 3. whatever {@link StorageDescriptorBuilder} requires. */ public class PartitionBuilder extends StorageDescriptorBuilder<PartitionBuilder> { - private String dbName, tableName; + private String catName, dbName, tableName; private int createTime, lastAccessTime; private Map<String, String> partParams; private List<String> values; @@ -40,6 +45,7 @@ public class PartitionBuilder extends StorageDescriptorBuilder<PartitionBuilder> // Set some reasonable defaults partParams = new HashMap<>(); createTime = lastAccessTime = (int)(System.currentTimeMillis() / 1000); + dbName = Warehouse.DEFAULT_DATABASE_NAME; super.setChild(this); } @@ -53,9 +59,10 @@ public class PartitionBuilder extends StorageDescriptorBuilder<PartitionBuilder> return this; } - public PartitionBuilder fromTable(Table table) { + public PartitionBuilder inTable(Table table) { this.dbName = table.getDbName(); this.tableName = table.getTableName(); + this.catName = table.getCatName(); setCols(table.getSd().getCols()); return this; } @@ -92,12 +99,21 @@ public class PartitionBuilder extends StorageDescriptorBuilder<PartitionBuilder> return this; } - public Partition build() throws MetaException { - if (dbName == null || tableName == null) { - throw new MetaException("database name and table name must be provided"); + public Partition build(Configuration conf) throws MetaException { + if (tableName == null) { + throw new MetaException("table name must be provided"); } if (values == null) throw new MetaException("You must provide partition values"); - return new Partition(values, dbName, tableName, createTime, lastAccessTime, buildSd(), + if (catName == null) catName = MetaStoreUtils.getDefaultCatalog(conf); + Partition p = new Partition(values, dbName, tableName, createTime, lastAccessTime, buildSd(), partParams); + p.setCatName(catName); + return p; + } + + public Partition addToTable(IMetaStoreClient client, Configuration conf) throws TException { + Partition p = build(conf); + client.add_partition(p); + return p; } } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLCheckConstraintBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLCheckConstraintBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLCheckConstraintBuilder.java new file mode 100644 index 0000000..ec99729 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLCheckConstraintBuilder.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; + +import java.util.ArrayList; +import java.util.List; + +public class SQLCheckConstraintBuilder extends ConstraintBuilder<SQLCheckConstraintBuilder> { + private String checkExpression; + + public SQLCheckConstraintBuilder() { + super.setChild(this); + } + + public SQLCheckConstraintBuilder setCheckExpression(String checkExpression) { + this.checkExpression = checkExpression; + return this; + } + + public List<SQLCheckConstraint> build(Configuration conf) throws MetaException { + if (checkExpression == null) { + throw new MetaException("check expression must be set"); + } + checkBuildable("check_constraint", conf); + List<SQLCheckConstraint> cc = new ArrayList<>(columns.size()); + for (String column : columns) { + cc.add(new SQLCheckConstraint(catName, dbName, tableName, column, checkExpression, + constraintName, enable, validate, rely)); + } + return cc; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLDefaultConstraintBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLDefaultConstraintBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLDefaultConstraintBuilder.java new file mode 100644 index 0000000..b24663d --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLDefaultConstraintBuilder.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.client.builder; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; + +import java.util.ArrayList; +import java.util.List; + +public class SQLDefaultConstraintBuilder extends ConstraintBuilder<SQLDefaultConstraintBuilder> { + private Object defaultVal; + + public SQLDefaultConstraintBuilder() { + super.setChild(this); + } + + public SQLDefaultConstraintBuilder setDefaultVal(Object defaultVal) { + this.defaultVal = defaultVal; + return this; + } + + public List<SQLDefaultConstraint> build(Configuration conf) throws MetaException { + if (defaultVal == null) { + throw new MetaException("default value must be set"); + } + checkBuildable("default_value", conf); + List<SQLDefaultConstraint> dv = new ArrayList<>(columns.size()); + for (String column : columns) { + dv.add(new SQLDefaultConstraint(catName, dbName, tableName, column, + defaultVal.toString(), constraintName, enable, validate, rely)); + } + return dv; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java index a39319a..f5adda1 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLForeignKeyBuilder.java @@ -17,21 +17,30 @@ */ package org.apache.hadoop.hive.metastore.client.builder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.SQLForeignKey; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import java.util.ArrayList; +import java.util.List; + /** * Builder for {@link SQLForeignKey}. Requires what {@link ConstraintBuilder} requires, plus * primary key * database, table, column and name. */ public class SQLForeignKeyBuilder extends ConstraintBuilder<SQLForeignKeyBuilder> { - private String pkDb, pkTable, pkColumn, pkName; + private String pkDb, pkTable, pkName; + private List<String> pkColumns; private int updateRule, deleteRule; public SQLForeignKeyBuilder() { + super.setChild(this); updateRule = deleteRule = 0; + pkColumns = new ArrayList<>(); + pkDb = Warehouse.DEFAULT_DATABASE_NAME; } public SQLForeignKeyBuilder setPkDb(String pkDb) { @@ -44,8 +53,8 @@ public class SQLForeignKeyBuilder extends ConstraintBuilder<SQLForeignKeyBuilder return this; } - public SQLForeignKeyBuilder setPkColumn(String pkColumn) { - this.pkColumn = pkColumn; + public SQLForeignKeyBuilder addPkColumn(String pkColumn) { + pkColumns.add(pkColumn); return this; } @@ -54,11 +63,11 @@ public class SQLForeignKeyBuilder extends ConstraintBuilder<SQLForeignKeyBuilder return this; } - public SQLForeignKeyBuilder setPrimaryKey(SQLPrimaryKey pk) { - pkDb = pk.getTable_db(); - pkTable = pk.getTable_name(); - pkColumn = pk.getColumn_name(); - pkName = pk.getPk_name(); + public SQLForeignKeyBuilder fromPrimaryKey(List<SQLPrimaryKey> pk) { + pkDb = pk.get(0).getTable_db(); + pkTable = pk.get(0).getTable_name(); + for (SQLPrimaryKey pkcol : pk) pkColumns.add(pkcol.getColumn_name()); + pkName = pk.get(0).getPk_name(); return this; } @@ -72,12 +81,23 @@ public class SQLForeignKeyBuilder extends ConstraintBuilder<SQLForeignKeyBuilder return this; } - public SQLForeignKey build() throws MetaException { - checkBuildable("foreign_key"); - if (pkDb == null || pkTable == null || pkColumn == null || pkName == null) { - throw new MetaException("You must provide the primary key database, table, column, and name"); + public List<SQLForeignKey> build(Configuration conf) throws MetaException { + checkBuildable("to_" + pkTable + "_foreign_key", conf); + if (pkTable == null || pkColumns.isEmpty() || pkName == null) { + throw new MetaException("You must provide the primary key table, columns, and name"); + } + if (columns.size() != pkColumns.size()) { + throw new MetaException("The number of foreign columns must match the number of primary key" + + " columns"); + } + List<SQLForeignKey> fk = new ArrayList<>(columns.size()); + for (int i = 0; i < columns.size(); i++) { + SQLForeignKey keyCol = new SQLForeignKey(pkDb, pkTable, pkColumns.get(i), dbName, tableName, + columns.get(i), getNextSeq(), updateRule, deleteRule, constraintName, pkName, enable, + validate, rely); + keyCol.setCatName(catName); + fk.add(keyCol); } - return new SQLForeignKey(pkDb, pkTable, pkColumn, dbName, tableName, columnName, keySeq, - updateRule, deleteRule, constraintName, pkName, enable, validate, rely); + return fk; } } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLNotNullConstraintBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLNotNullConstraintBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLNotNullConstraintBuilder.java index 77d1e49..497032e 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLNotNullConstraintBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLNotNullConstraintBuilder.java @@ -17,9 +17,13 @@ */ package org.apache.hadoop.hive.metastore.client.builder; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; +import java.util.ArrayList; +import java.util.List; + /** * Builder for {@link SQLNotNullConstraint}. Only requires what {@link ConstraintBuilder} requires. */ @@ -29,9 +33,20 @@ public class SQLNotNullConstraintBuilder extends ConstraintBuilder<SQLNotNullCon super.setChild(this); } - public SQLNotNullConstraint build() throws MetaException { - checkBuildable("not_null_constraint"); - return new SQLNotNullConstraint(dbName, tableName, columnName, constraintName, enable, - validate, rely); + public SQLNotNullConstraintBuilder setColName(String colName) { + assert columns.isEmpty(); + columns.add(colName); + return this; + } + + public List<SQLNotNullConstraint> build(Configuration conf) throws MetaException { + checkBuildable("not_null_constraint", conf); + List<SQLNotNullConstraint> uc = new ArrayList<>(columns.size()); + for (String column : columns) { + SQLNotNullConstraint c = new SQLNotNullConstraint(catName, dbName, tableName, columns.get(0), + constraintName, enable, validate, rely); + uc.add(c); + } + return uc; } } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java index 9000f86..40f74bd 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLPrimaryKeyBuilder.java @@ -17,9 +17,13 @@ */ package org.apache.hadoop.hive.metastore.client.builder; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import java.util.ArrayList; +import java.util.List; + /** * Builder for {@link SQLPrimaryKey}. Only requires what {@link ConstraintBuilder} requires. */ @@ -34,9 +38,15 @@ public class SQLPrimaryKeyBuilder extends ConstraintBuilder<SQLPrimaryKeyBuilder return setConstraintName(name); } - public SQLPrimaryKey build() throws MetaException { - checkBuildable("primary_key"); - return new SQLPrimaryKey(dbName, tableName, columnName, keySeq, constraintName, enable, - validate, rely); + public List<SQLPrimaryKey> build(Configuration conf) throws MetaException { + checkBuildable("primary_key", conf); + List<SQLPrimaryKey> pk = new ArrayList<>(columns.size()); + for (String colName : columns) { + SQLPrimaryKey keyCol = new SQLPrimaryKey(dbName, tableName, colName, getNextSeq(), + constraintName, enable, validate, rely); + keyCol.setCatName(catName); + pk.add(keyCol); + } + return pk; } } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLUniqueConstraintBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLUniqueConstraintBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLUniqueConstraintBuilder.java index 640e9d1..138ee15 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLUniqueConstraintBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SQLUniqueConstraintBuilder.java @@ -17,9 +17,13 @@ */ package org.apache.hadoop.hive.metastore.client.builder; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import java.util.ArrayList; +import java.util.List; + /** * Builder for {@link SQLUniqueConstraint}. Only requires what {@link ConstraintBuilder} requires. */ @@ -29,9 +33,14 @@ public class SQLUniqueConstraintBuilder extends ConstraintBuilder<SQLUniqueConst super.setChild(this); } - public SQLUniqueConstraint build() throws MetaException { - checkBuildable("unique_constraint"); - return new SQLUniqueConstraint(dbName, tableName, columnName, keySeq, constraintName, enable, - validate, rely); + public List<SQLUniqueConstraint> build(Configuration conf) throws MetaException { + checkBuildable("unique_constraint", conf); + List<SQLUniqueConstraint> uc = new ArrayList<>(columns.size()); + for (String column : columns) { + SQLUniqueConstraint c = new SQLUniqueConstraint(catName, dbName, tableName, column, getNextSeq(), + constraintName, enable, validate, rely); + uc.add(c); + } + return uc; } } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SchemaVersionBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SchemaVersionBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SchemaVersionBuilder.java index ceb0f49..521be3e 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SchemaVersionBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/SchemaVersionBuilder.java @@ -23,8 +23,11 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.SchemaVersion; import org.apache.hadoop.hive.metastore.api.SchemaVersionState; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; + public class SchemaVersionBuilder extends SerdeAndColsBuilder<SchemaVersionBuilder> { - private String schemaName, dbName; // required + private String schemaName, dbName, catName; // required private int version; // required private long createdAt; // required private SchemaVersionState state; // optional @@ -34,6 +37,8 @@ public class SchemaVersionBuilder extends SerdeAndColsBuilder<SchemaVersionBuild private String name; // optional public SchemaVersionBuilder() { + catName = DEFAULT_CATALOG_NAME; + dbName = DEFAULT_DATABASE_NAME; createdAt = System.currentTimeMillis() / 1000; version = -1; super.setChild(this); @@ -50,6 +55,7 @@ public class SchemaVersionBuilder extends SerdeAndColsBuilder<SchemaVersionBuild } public SchemaVersionBuilder versionOf(ISchema schema) { + this.catName = schema.getCatName(); this.dbName = schema.getDbName(); this.schemaName = schema.getName(); return this; @@ -92,11 +98,11 @@ public class SchemaVersionBuilder extends SerdeAndColsBuilder<SchemaVersionBuild } public SchemaVersion build() throws MetaException { - if (schemaName == null || dbName == null || version < 0) { - throw new MetaException("You must provide the database name, schema name, and schema version"); + if (schemaName == null || version < 0) { + throw new MetaException("You must provide the schema name, and schema version"); } SchemaVersion schemaVersion = - new SchemaVersion(new ISchemaName(dbName, schemaName), version, createdAt, getCols()); + new SchemaVersion(new ISchemaName(catName, dbName, schemaName), version, createdAt, getCols()); if (state != null) schemaVersion.setState(state); if (description != null) schemaVersion.setDescription(description); if (schemaText != null) schemaVersion.setSchemaText(schemaText); http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java index 2b9f816..79ef7de 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/client/builder/TableBuilder.java @@ -17,48 +17,69 @@ */ package org.apache.hadoop.hive.metastore.client.builder; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.ValidTxnList; +import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.BasicTxnInfo; +import org.apache.hadoop.hive.metastore.api.CreationMetadata; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; +import org.apache.thrift.TException; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** * Build a {@link Table}. The database name and table name must be provided, plus whatever is * needed by the underlying {@link StorageDescriptorBuilder}. */ public class TableBuilder extends StorageDescriptorBuilder<TableBuilder> { - private String dbName, tableName, owner, viewOriginalText, viewExpandedText, type; + private String catName, dbName, tableName, owner, viewOriginalText, viewExpandedText, type, + mvValidTxnList; private List<FieldSchema> partCols; private int createTime, lastAccessTime, retention; private Map<String, String> tableParams; private boolean rewriteEnabled, temporary; + private Set<String> mvReferencedTables; + public TableBuilder() { // Set some reasonable defaults + dbName = Warehouse.DEFAULT_DATABASE_NAME; tableParams = new HashMap<>(); createTime = lastAccessTime = (int)(System.currentTimeMillis() / 1000); retention = 0; partCols = new ArrayList<>(); type = TableType.MANAGED_TABLE.name(); + mvReferencedTables = new HashSet<>(); + temporary = false; super.setChild(this); } + public TableBuilder setCatName(String catName) { + this.catName = catName; + return this; + } + public TableBuilder setDbName(String dbName) { this.dbName = dbName; return this; } - public TableBuilder setDbName(Database db) { + public TableBuilder inDb(Database db) { this.dbName = db.getName(); + this.catName = db.getCatalogName(); return this; } @@ -139,9 +160,19 @@ public class TableBuilder extends StorageDescriptorBuilder<TableBuilder> { return this; } - public Table build() throws MetaException { - if (dbName == null || tableName == null) { - throw new MetaException("You must set the database and table name"); + public TableBuilder addMaterializedViewReferencedTable(String tableName) { + mvReferencedTables.add(tableName); + return this; + } + + public TableBuilder setMaterializedViewValidTxnList(ValidTxnList validTxnList) { + mvValidTxnList = validTxnList.writeToString(); + return this; + } + + public Table build(Configuration conf) throws MetaException { + if (tableName == null) { + throw new MetaException("You must set the table name"); } if (owner == null) { try { @@ -150,15 +181,24 @@ public class TableBuilder extends StorageDescriptorBuilder<TableBuilder> { throw MetaStoreUtils.newMetaException(e); } } + if (catName == null) catName = MetaStoreUtils.getDefaultCatalog(conf); Table t = new Table(tableName, dbName, owner, createTime, lastAccessTime, retention, buildSd(), partCols, tableParams, viewOriginalText, viewExpandedText, type); - if (rewriteEnabled) { - t.setRewriteEnabled(true); - } - if (temporary) { - t.setTemporary(temporary); + if (rewriteEnabled) t.setRewriteEnabled(true); + if (temporary) t.setTemporary(temporary); + t.setCatName(catName); + if (!mvReferencedTables.isEmpty()) { + CreationMetadata cm = new CreationMetadata(catName, dbName, tableName, mvReferencedTables); + if (mvValidTxnList != null) cm.setValidTxnList(mvValidTxnList); + t.setCreationMetadata(cm); } return t; } + public Table create(IMetaStoreClient client, Configuration conf) throws TException { + Table t = build(conf); + client.createTable(t); + return t; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index b8976ed..995137f 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -306,6 +306,12 @@ public class MetastoreConf { CAPABILITY_CHECK("metastore.client.capability.check", "hive.metastore.client.capability.check", true, "Whether to check client capabilities for potentially breaking API usage."), + CATALOG_DEFAULT("metastore.catalog.default", "metastore.catalog.default", "hive", + "The default catalog to use when a catalog is not specified. Default is 'hive' (the " + + "default catalog)."), + CATALOGS_TO_CACHE("metastore.cached.rawstore.catalogs", "metastore.cached.rawstore.catalogs", + "hive", "Comma separated list of catalogs to cache in the CachedStore. Default is 'hive' " + + "(the default catalog). Empty string means all catalogs will be cached."), CLIENT_CONNECT_RETRY_DELAY("metastore.client.connect.retry.delay", "hive.metastore.client.connect.retry.delay", 1, TimeUnit.SECONDS, "Number of seconds for the client to wait between consecutive connection attempts"), http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateCatalogEvent.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateCatalogEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateCatalogEvent.java new file mode 100644 index 0000000..e667277 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/CreateCatalogEvent.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; +import org.apache.hadoop.hive.metastore.api.Catalog; + [email protected] [email protected] +public class CreateCatalogEvent extends ListenerEvent { + + private final Catalog cat; + + public CreateCatalogEvent(boolean status, IHMSHandler handler, Catalog cat) { + super(status, handler); + this.cat = cat; + } + + public Catalog getCatalog() { + return cat; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropCatalogEvent.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropCatalogEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropCatalogEvent.java new file mode 100644 index 0000000..67c6d51 --- /dev/null +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropCatalogEvent.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; +import org.apache.hadoop.hive.metastore.api.Catalog; + [email protected] [email protected] +public class DropCatalogEvent extends ListenerEvent { + + private final Catalog cat; + + public DropCatalogEvent(boolean status, IHMSHandler handler, Catalog cat) { + super(status, handler); + this.cat = cat; + } + + public Catalog getCatalog() { + return cat; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java index 4c5918f..ccd968b 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/events/DropConstraintEvent.java @@ -26,17 +26,23 @@ import org.apache.hadoop.hive.metastore.IHMSHandler; @InterfaceStability.Stable public class DropConstraintEvent extends ListenerEvent { + private final String catName; private final String dbName; private final String tableName; private final String constraintName; - public DropConstraintEvent(String dbName, String tableName, String constraintName, + public DropConstraintEvent(String catName, String dbName, String tableName, String constraintName, boolean status, IHMSHandler handler) { super(status, handler); + this.catName = catName; this.dbName = dbName; this.tableName = tableName; this.constraintName = constraintName; } + public String getCatName() { + return catName; + } + public String getDbName() { return dbName; }
