http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java index a72fc0b..d451f96 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java @@ -28,9 +28,11 @@ import java.util.concurrent.Executors; import java.util.concurrent.ThreadFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog; +import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.TableType; +import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; @@ -46,6 +48,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -54,16 +57,19 @@ import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; + @Category(MetastoreCheckinTest.class) public class TestCachedStore { private ObjectStore objectStore; private CachedStore cachedStore; private SharedCache sharedCache; + private Configuration conf; @Before public void setUp() throws Exception { - Configuration conf = MetastoreConf.newMetastoreConf(); + conf = MetastoreConf.newMetastoreConf(); MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true); MetaStoreTestUtils.setConfForStandloneMode(conf); objectStore = new ObjectStore(); @@ -76,6 +82,9 @@ public class TestCachedStore { sharedCache.getDatabaseCache().clear(); sharedCache.getTableCache().clear(); sharedCache.getSdCache().clear(); + + // Create the 'hive' catalog + HiveMetaStore.HMSHandler.createDefaultCatalog(objectStore, new Warehouse(conf)); } /********************************************************************************************** @@ -89,67 +98,67 @@ public class TestCachedStore { String dbOwner = "user1"; Database db = createTestDb(dbName, dbOwner); objectStore.createDatabase(db); - db = objectStore.getDatabase(dbName); + db = objectStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); // Prewarm CachedStore CachedStore.setCachePrewarmedState(false); CachedStore.prewarm(objectStore); // Read database via CachedStore - Database dbRead = cachedStore.getDatabase(dbName); + Database dbRead = cachedStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); Assert.assertEquals(db, dbRead); // Add another db via CachedStore final String dbName1 = "testDatabaseOps1"; Database db1 = createTestDb(dbName1, dbOwner); cachedStore.createDatabase(db1); - db1 = cachedStore.getDatabase(dbName1); + db1 = cachedStore.getDatabase(DEFAULT_CATALOG_NAME, dbName1); // Read db via ObjectStore - dbRead = objectStore.getDatabase(dbName1); + dbRead = objectStore.getDatabase(DEFAULT_CATALOG_NAME, dbName1); Assert.assertEquals(db1, dbRead); // Alter the db via CachedStore (can only alter owner or parameters) dbOwner = "user2"; db = new Database(db); db.setOwnerName(dbOwner); - cachedStore.alterDatabase(dbName, db); - db = cachedStore.getDatabase(dbName); + cachedStore.alterDatabase(DEFAULT_CATALOG_NAME, dbName, db); + db = cachedStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); // Read db via ObjectStore - dbRead = objectStore.getDatabase(dbName); + dbRead = objectStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); Assert.assertEquals(db, dbRead); // Add another db via ObjectStore final String dbName2 = "testDatabaseOps2"; Database db2 = createTestDb(dbName2, dbOwner); objectStore.createDatabase(db2); - db2 = objectStore.getDatabase(dbName2); + db2 = objectStore.getDatabase(DEFAULT_CATALOG_NAME, dbName2); // Alter db "testDatabaseOps" via ObjectStore dbOwner = "user1"; db = new Database(db); db.setOwnerName(dbOwner); - objectStore.alterDatabase(dbName, db); - db = objectStore.getDatabase(dbName); + objectStore.alterDatabase(DEFAULT_CATALOG_NAME, dbName, db); + db = objectStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); // Drop db "testDatabaseOps1" via ObjectStore - objectStore.dropDatabase(dbName1); + objectStore.dropDatabase(DEFAULT_CATALOG_NAME, dbName1); // We update twice to accurately detect if cache is dirty or not updateCache(cachedStore); updateCache(cachedStore); // Read the newly added db via CachedStore - dbRead = cachedStore.getDatabase(dbName2); + dbRead = cachedStore.getDatabase(DEFAULT_CATALOG_NAME, dbName2); Assert.assertEquals(db2, dbRead); // Read the altered db via CachedStore (altered user from "user2" to "user1") - dbRead = cachedStore.getDatabase(dbName); + dbRead = cachedStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); Assert.assertEquals(db, dbRead); // Try to read the dropped db after cache update try { - dbRead = cachedStore.getDatabase(dbName1); + dbRead = cachedStore.getDatabase(DEFAULT_CATALOG_NAME, dbName1); Assert.fail("The database: " + dbName1 + " should have been removed from the cache after running the update service"); } catch (NoSuchObjectException e) { @@ -157,8 +166,8 @@ public class TestCachedStore { } // Clean up - objectStore.dropDatabase(dbName); - objectStore.dropDatabase(dbName2); + objectStore.dropDatabase(DEFAULT_CATALOG_NAME, dbName); + objectStore.dropDatabase(DEFAULT_CATALOG_NAME, dbName2); sharedCache.getDatabaseCache().clear(); sharedCache.getTableCache().clear(); sharedCache.getSdCache().clear(); @@ -171,7 +180,7 @@ public class TestCachedStore { String dbOwner = "user1"; Database db = createTestDb(dbName, dbOwner); objectStore.createDatabase(db); - db = objectStore.getDatabase(dbName); + db = objectStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); // Add a table via ObjectStore String tblName = "tbl"; @@ -184,16 +193,16 @@ public class TestCachedStore { List<FieldSchema> ptnCols = new ArrayList<FieldSchema>(); Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols); objectStore.createTable(tbl); - tbl = objectStore.getTable(dbName, tblName); + tbl = objectStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); // Prewarm CachedStore CachedStore.setCachePrewarmedState(false); CachedStore.prewarm(objectStore); // Read database, table via CachedStore - Database dbRead= cachedStore.getDatabase(dbName); + Database dbRead= cachedStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); Assert.assertEquals(db, dbRead); - Table tblRead = cachedStore.getTable(dbName, tblName); + Table tblRead = cachedStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); Assert.assertEquals(tbl, tblRead); // Add a new table via CachedStore @@ -201,10 +210,10 @@ public class TestCachedStore { Table tbl1 = new Table(tbl); tbl1.setTableName(tblName1); cachedStore.createTable(tbl1); - tbl1 = cachedStore.getTable(dbName, tblName1); + tbl1 = cachedStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName1); // Read via object store - tblRead = objectStore.getTable(dbName, tblName1); + tblRead = objectStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName1); Assert.assertEquals(tbl1, tblRead); // Add a new table via ObjectStore @@ -212,43 +221,43 @@ public class TestCachedStore { Table tbl2 = new Table(tbl); tbl2.setTableName(tblName2); objectStore.createTable(tbl2); - tbl2 = objectStore.getTable(dbName, tblName2); + tbl2 = objectStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName2); // Alter table "tbl" via ObjectStore tblOwner = "user2"; tbl.setOwner(tblOwner); - objectStore.alterTable(dbName, tblName, tbl); - tbl = objectStore.getTable(dbName, tblName); + objectStore.alterTable(DEFAULT_CATALOG_NAME, dbName, tblName, tbl); + tbl = objectStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); // Drop table "tbl1" via ObjectStore - objectStore.dropTable(dbName, tblName1); + objectStore.dropTable(DEFAULT_CATALOG_NAME, dbName, tblName1); // We update twice to accurately detect if cache is dirty or not updateCache(cachedStore); updateCache(cachedStore); // Read "tbl2" via CachedStore - tblRead = cachedStore.getTable(dbName, tblName2); + tblRead = cachedStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName2); Assert.assertEquals(tbl2, tblRead); // Read the altered "tbl" via CachedStore - tblRead = cachedStore.getTable(dbName, tblName); + tblRead = cachedStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); Assert.assertEquals(tbl, tblRead); // Try to read the dropped "tbl1" via CachedStore (should throw exception) - tblRead = cachedStore.getTable(dbName, tblName1); + tblRead = cachedStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName1); Assert.assertNull(tblRead); // Should return "tbl" and "tbl2" - List<String> tblNames = cachedStore.getTables(dbName, "*"); + List<String> tblNames = cachedStore.getTables(DEFAULT_CATALOG_NAME, dbName, "*"); Assert.assertTrue(tblNames.contains(tblName)); Assert.assertTrue(!tblNames.contains(tblName1)); Assert.assertTrue(tblNames.contains(tblName2)); // Clean up - objectStore.dropTable(dbName, tblName); - objectStore.dropTable(dbName, tblName2); - objectStore.dropDatabase(dbName); + objectStore.dropTable(DEFAULT_CATALOG_NAME, dbName, tblName); + objectStore.dropTable(DEFAULT_CATALOG_NAME, dbName, tblName2); + objectStore.dropDatabase(DEFAULT_CATALOG_NAME, dbName); sharedCache.getDatabaseCache().clear(); sharedCache.getTableCache().clear(); sharedCache.getSdCache().clear(); @@ -261,7 +270,7 @@ public class TestCachedStore { String dbOwner = "user1"; Database db = createTestDb(dbName, dbOwner); objectStore.createDatabase(db); - db = objectStore.getDatabase(dbName); + db = objectStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); // Add a table via ObjectStore String tblName = "tbl"; @@ -276,76 +285,81 @@ public class TestCachedStore { ptnCols.add(ptnCol1); Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols); objectStore.createTable(tbl); - tbl = objectStore.getTable(dbName, tblName); + tbl = objectStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); final String ptnColVal1 = "aaa"; Map<String, String> partParams = new HashMap<String, String>(); Partition ptn1 = new Partition(Arrays.asList(ptnColVal1), dbName, tblName, 0, 0, tbl.getSd(), partParams); + ptn1.setCatName(DEFAULT_CATALOG_NAME); objectStore.addPartition(ptn1); - ptn1 = objectStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal1)); + ptn1 = objectStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1)); + ptn1.setCatName(DEFAULT_CATALOG_NAME); final String ptnColVal2 = "bbb"; Partition ptn2 = new Partition(Arrays.asList(ptnColVal2), dbName, tblName, 0, 0, tbl.getSd(), partParams); + ptn2.setCatName(DEFAULT_CATALOG_NAME); objectStore.addPartition(ptn2); - ptn2 = objectStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal2)); + ptn2 = objectStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal2)); // Prewarm CachedStore CachedStore.setCachePrewarmedState(false); CachedStore.prewarm(objectStore); // Read database, table, partition via CachedStore - Database dbRead = cachedStore.getDatabase(dbName); + Database dbRead = cachedStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); Assert.assertEquals(db, dbRead); - Table tblRead = cachedStore.getTable(dbName, tblName); + Table tblRead = cachedStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); Assert.assertEquals(tbl, tblRead); - Partition ptn1Read = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal1)); + Partition ptn1Read = cachedStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1)); Assert.assertEquals(ptn1, ptn1Read); - Partition ptn2Read = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal2)); + Partition ptn2Read = cachedStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal2)); Assert.assertEquals(ptn2, ptn2Read); // Add a new partition via ObjectStore final String ptnColVal3 = "ccc"; Partition ptn3 = new Partition(Arrays.asList(ptnColVal3), dbName, tblName, 0, 0, tbl.getSd(), partParams); + ptn3.setCatName(DEFAULT_CATALOG_NAME); objectStore.addPartition(ptn3); - ptn3 = objectStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal3)); + ptn3 = objectStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal3)); // Alter an existing partition ("aaa") via ObjectStore final String ptnColVal1Alt = "aaaAlt"; Partition ptn1Atl = new Partition(Arrays.asList(ptnColVal1Alt), dbName, tblName, 0, 0, tbl.getSd(), partParams); - objectStore.alterPartition(dbName, tblName, Arrays.asList(ptnColVal1), ptn1Atl); - ptn1Atl = objectStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal1Alt)); + ptn1Atl.setCatName(DEFAULT_CATALOG_NAME); + objectStore.alterPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1), ptn1Atl); + ptn1Atl = objectStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1Alt)); // Drop an existing partition ("bbb") via ObjectStore - objectStore.dropPartition(dbName, tblName, Arrays.asList(ptnColVal2)); + objectStore.dropPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal2)); // We update twice to accurately detect if cache is dirty or not updateCache(cachedStore); updateCache(cachedStore); // Read the newly added partition via CachedStore - Partition ptnRead = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal3)); + Partition ptnRead = cachedStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal3)); Assert.assertEquals(ptn3, ptnRead); // Read the altered partition via CachedStore - ptnRead = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal1Alt)); + ptnRead = cachedStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1Alt)); Assert.assertEquals(ptn1Atl, ptnRead); // Try to read the dropped partition via CachedStore try { - ptnRead = cachedStore.getPartition(dbName, tblName, Arrays.asList(ptnColVal2)); + ptnRead = cachedStore.getPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal2)); Assert.fail("The partition: " + ptnColVal2 + " should have been removed from the cache after running the update service"); } catch (NoSuchObjectException e) { // Expected } // Clean up - objectStore.dropPartition(dbName, tblName, Arrays.asList(ptnColVal1Alt)); - objectStore.dropPartition(dbName, tblName, Arrays.asList(ptnColVal3)); - objectStore.dropTable(dbName, tblName); - objectStore.dropDatabase(dbName); + objectStore.dropPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal1Alt)); + objectStore.dropPartition(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(ptnColVal3)); + objectStore.dropTable(DEFAULT_CATALOG_NAME, dbName, tblName); + objectStore.dropDatabase(DEFAULT_CATALOG_NAME, dbName); sharedCache.getDatabaseCache().clear(); sharedCache.getTableCache().clear(); sharedCache.getSdCache().clear(); @@ -358,7 +372,7 @@ public class TestCachedStore { String dbOwner = "user1"; Database db = createTestDb(dbName, dbOwner); objectStore.createDatabase(db); - db = objectStore.getDatabase(dbName); + db = objectStore.getDatabase(DEFAULT_CATALOG_NAME, dbName); // Add a table via ObjectStore final String tblName = "tbl"; @@ -389,7 +403,7 @@ public class TestCachedStore { ptnCols.add(ptnCol1); Table tbl = createTestTbl(dbName, tblName, tblOwner, cols, ptnCols); objectStore.createTable(tbl); - tbl = objectStore.getTable(dbName, tblName); + tbl = objectStore.getTable(DEFAULT_CATALOG_NAME, dbName, tblName); // Add ColumnStatistics for tbl to metastore DB via ObjectStore ColumnStatistics stats = new ColumnStatistics(); @@ -440,13 +454,13 @@ public class TestCachedStore { // Read table stats via CachedStore ColumnStatistics newStats = - cachedStore.getTableColumnStatistics(dbName, tblName, + cachedStore.getTableColumnStatistics(DEFAULT_CATALOG_NAME, dbName, tblName, Arrays.asList(col1.getName(), col2.getName(), col3.getName())); Assert.assertEquals(stats, newStats); // Clean up - objectStore.dropTable(dbName, tblName); - objectStore.dropDatabase(dbName); + objectStore.dropTable(DEFAULT_CATALOG_NAME, dbName, tblName); + objectStore.dropDatabase(DEFAULT_CATALOG_NAME, dbName); sharedCache.getDatabaseCache().clear(); sharedCache.getTableCache().clear(); sharedCache.getSdCache().clear(); @@ -466,11 +480,11 @@ public class TestCachedStore { sharedCache.addDatabaseToCache(db2); sharedCache.addDatabaseToCache(db3); Assert.assertEquals(sharedCache.getCachedDatabaseCount(), 3); - sharedCache.alterDatabaseInCache("db1", newDb1); + sharedCache.alterDatabaseInCache(DEFAULT_CATALOG_NAME, "db1", newDb1); Assert.assertEquals(sharedCache.getCachedDatabaseCount(), 3); - sharedCache.removeDatabaseFromCache("db2"); + sharedCache.removeDatabaseFromCache(DEFAULT_CATALOG_NAME, "db2"); Assert.assertEquals(sharedCache.getCachedDatabaseCount(), 2); - List<String> dbs = sharedCache.listCachedDatabases(); + List<String> dbs = sharedCache.listCachedDatabases(DEFAULT_CATALOG_NAME); Assert.assertEquals(dbs.size(), 2); Assert.assertTrue(dbs.contains("newdb1")); Assert.assertTrue(dbs.contains("db3")); @@ -528,26 +542,26 @@ public class TestCachedStore { newTbl1.setSd(newSd1); newTbl1.setPartitionKeys(new ArrayList<>()); - sharedCache.addTableToCache("db1", "tbl1", tbl1); - sharedCache.addTableToCache("db1", "tbl2", tbl2); - sharedCache.addTableToCache("db1", "tbl3", tbl3); - sharedCache.addTableToCache("db2", "tbl1", tbl1); + sharedCache.addTableToCache(DEFAULT_CATALOG_NAME, "db1", "tbl1", tbl1); + sharedCache.addTableToCache(DEFAULT_CATALOG_NAME, "db1", "tbl2", tbl2); + sharedCache.addTableToCache(DEFAULT_CATALOG_NAME, "db1", "tbl3", tbl3); + sharedCache.addTableToCache(DEFAULT_CATALOG_NAME, "db2", "tbl1", tbl1); Assert.assertEquals(sharedCache.getCachedTableCount(), 4); Assert.assertEquals(sharedCache.getSdCache().size(), 2); - Table t = sharedCache.getTableFromCache("db1", "tbl1"); + Table t = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME, "db1", "tbl1"); Assert.assertEquals(t.getSd().getLocation(), "loc1"); - sharedCache.removeTableFromCache("db1", "tbl1"); + sharedCache.removeTableFromCache(DEFAULT_CATALOG_NAME, "db1", "tbl1"); Assert.assertEquals(sharedCache.getCachedTableCount(), 3); Assert.assertEquals(sharedCache.getSdCache().size(), 2); - sharedCache.alterTableInCache("db2", "tbl1", newTbl1); + sharedCache.alterTableInCache(DEFAULT_CATALOG_NAME, "db2", "tbl1", newTbl1); Assert.assertEquals(sharedCache.getCachedTableCount(), 3); Assert.assertEquals(sharedCache.getSdCache().size(), 3); - sharedCache.removeTableFromCache("db1", "tbl2"); + sharedCache.removeTableFromCache(DEFAULT_CATALOG_NAME, "db1", "tbl2"); Assert.assertEquals(sharedCache.getCachedTableCount(), 2); Assert.assertEquals(sharedCache.getSdCache().size(), 2); } @@ -568,9 +582,9 @@ public class TestCachedStore { cols.add(col2); List<FieldSchema> ptnCols = new ArrayList<FieldSchema>(); Table tbl1 = createTestTbl(dbName, tbl1Name, owner, cols, ptnCols); - sharedCache.addTableToCache(dbName, tbl1Name, tbl1); + sharedCache.addTableToCache(DEFAULT_CATALOG_NAME, dbName, tbl1Name, tbl1); Table tbl2 = createTestTbl(dbName, tbl2Name, owner, cols, ptnCols); - sharedCache.addTableToCache(dbName, tbl2Name, tbl2); + sharedCache.addTableToCache(DEFAULT_CATALOG_NAME, dbName, tbl2Name, tbl2); Partition part1 = new Partition(); StorageDescriptor sd1 = new StorageDescriptor(); @@ -622,20 +636,20 @@ public class TestCachedStore { newPart1.setSd(newSd1); newPart1.setValues(Arrays.asList("201701")); - sharedCache.addPartitionToCache(dbName, tbl1Name, part1); - sharedCache.addPartitionToCache(dbName, tbl1Name, part2); - sharedCache.addPartitionToCache(dbName, tbl1Name, part3); - sharedCache.addPartitionToCache(dbName, tbl2Name, part1); + sharedCache.addPartitionToCache(DEFAULT_CATALOG_NAME, dbName, tbl1Name, part1); + sharedCache.addPartitionToCache(DEFAULT_CATALOG_NAME, dbName, tbl1Name, part2); + sharedCache.addPartitionToCache(DEFAULT_CATALOG_NAME, dbName, tbl1Name, part3); + sharedCache.addPartitionToCache(DEFAULT_CATALOG_NAME, dbName, tbl2Name, part1); - Partition t = sharedCache.getPartitionFromCache(dbName, tbl1Name, Arrays.asList("201701")); + Partition t = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME, dbName, tbl1Name, Arrays.asList("201701")); Assert.assertEquals(t.getSd().getLocation(), "loc1"); - sharedCache.removePartitionFromCache(dbName, tbl2Name, Arrays.asList("201701")); - t = sharedCache.getPartitionFromCache(dbName, tbl2Name, Arrays.asList("201701")); + sharedCache.removePartitionFromCache(DEFAULT_CATALOG_NAME, dbName, tbl2Name, Arrays.asList("201701")); + t = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME, dbName, tbl2Name, Arrays.asList("201701")); Assert.assertNull(t); - sharedCache.alterPartitionInCache(dbName, tbl1Name, Arrays.asList("201701"), newPart1); - t = sharedCache.getPartitionFromCache(dbName, tbl1Name, Arrays.asList("201701")); + sharedCache.alterPartitionInCache(DEFAULT_CATALOG_NAME, dbName, tbl1Name, Arrays.asList("201701"), newPart1); + t = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME, dbName, tbl1Name, Arrays.asList("201701")); Assert.assertEquals(t.getSd().getLocation(), "loc1new"); } @@ -645,7 +659,10 @@ public class TestCachedStore { String tblName = "tbl"; String colName = "f1"; - Database db = new Database(dbName, null, "some_location", null); + Database db = new DatabaseBuilder() + .setName(dbName) + .setLocation("some_location") + .build(conf); cachedStore.createDatabase(db); List<FieldSchema> cols = new ArrayList<>(); @@ -659,6 +676,7 @@ public class TestCachedStore { Table tbl = new Table(tblName, dbName, null, 0, 0, 0, sd, partCols, new HashMap<>(), null, null, TableType.MANAGED_TABLE.toString()); + tbl.setCatName(DEFAULT_CATALOG_NAME); cachedStore.createTable(tbl); List<String> partVals1 = new ArrayList<>(); @@ -668,9 +686,11 @@ public class TestCachedStore { Partition ptn1 = new Partition(partVals1, dbName, tblName, 0, 0, sd, new HashMap<>()); + ptn1.setCatName(DEFAULT_CATALOG_NAME); cachedStore.addPartition(ptn1); Partition ptn2 = new Partition(partVals2, dbName, tblName, 0, 0, sd, new HashMap<>()); + ptn2.setCatName(DEFAULT_CATALOG_NAME); cachedStore.addPartition(ptn2); ColumnStatistics stats = new ColumnStatistics(); @@ -699,9 +719,9 @@ public class TestCachedStore { List<String> aggrPartVals = new ArrayList<>(); aggrPartVals.add("1"); aggrPartVals.add("2"); - AggrStats aggrStats = cachedStore.get_aggr_stats_for(dbName, tblName, aggrPartVals, colNames); + AggrStats aggrStats = cachedStore.get_aggr_stats_for(DEFAULT_CATALOG_NAME, dbName, tblName, aggrPartVals, colNames); Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumNulls(), 100); - aggrStats = cachedStore.get_aggr_stats_for(dbName, tblName, aggrPartVals, colNames); + aggrStats = cachedStore.get_aggr_stats_for(DEFAULT_CATALOG_NAME, dbName, tblName, aggrPartVals, colNames); Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumNulls(), 100); } @@ -712,6 +732,7 @@ public class TestCachedStore { String colName = "f1"; Database db = new Database(dbName, null, "some_location", null); + db.setCatalogName(DEFAULT_CATALOG_NAME); cachedStore.createDatabase(db); List<FieldSchema> cols = new ArrayList<>(); @@ -725,6 +746,7 @@ public class TestCachedStore { Table tbl = new Table(tblName, dbName, null, 0, 0, 0, sd, partCols, new HashMap<>(), null, null, TableType.MANAGED_TABLE.toString()); + tbl.setCatName(DEFAULT_CATALOG_NAME); cachedStore.createTable(tbl); List<String> partVals1 = new ArrayList<>(); @@ -734,9 +756,11 @@ public class TestCachedStore { Partition ptn1 = new Partition(partVals1, dbName, tblName, 0, 0, sd, new HashMap<>()); + ptn1.setCatName(DEFAULT_CATALOG_NAME); cachedStore.addPartition(ptn1); Partition ptn2 = new Partition(partVals2, dbName, tblName, 0, 0, sd, new HashMap<>()); + ptn2.setCatName(DEFAULT_CATALOG_NAME); cachedStore.addPartition(ptn2); ColumnStatistics stats = new ColumnStatistics(); @@ -767,10 +791,10 @@ public class TestCachedStore { List<String> aggrPartVals = new ArrayList<>(); aggrPartVals.add("1"); aggrPartVals.add("2"); - AggrStats aggrStats = cachedStore.get_aggr_stats_for(dbName, tblName, aggrPartVals, colNames); + AggrStats aggrStats = cachedStore.get_aggr_stats_for(DEFAULT_CATALOG_NAME, dbName, tblName, aggrPartVals, colNames); Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumNulls(), 100); Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumDVs(), 40); - aggrStats = cachedStore.get_aggr_stats_for(dbName, tblName, aggrPartVals, colNames); + aggrStats = cachedStore.get_aggr_stats_for(DEFAULT_CATALOG_NAME, dbName, tblName, aggrPartVals, colNames); Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumNulls(), 100); Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumDVs(), 40); } @@ -782,6 +806,7 @@ public class TestCachedStore { String colName = "f1"; Database db = new Database(dbName, null, "some_location", null); + db.setCatalogName(DEFAULT_CATALOG_NAME); cachedStore.createDatabase(db); List<FieldSchema> cols = new ArrayList<>(); @@ -795,6 +820,7 @@ public class TestCachedStore { Table tbl = new Table(tblName, dbName, null, 0, 0, 0, sd, partCols, new HashMap<>(), null, null, TableType.MANAGED_TABLE.toString()); + tbl.setCatName(DEFAULT_CATALOG_NAME); cachedStore.createTable(tbl); List<String> partVals1 = new ArrayList<>(); @@ -804,9 +830,11 @@ public class TestCachedStore { Partition ptn1 = new Partition(partVals1, dbName, tblName, 0, 0, sd, new HashMap<>()); + ptn1.setCatName(DEFAULT_CATALOG_NAME); cachedStore.addPartition(ptn1); Partition ptn2 = new Partition(partVals2, dbName, tblName, 0, 0, sd, new HashMap<>()); + ptn2.setCatName(DEFAULT_CATALOG_NAME); cachedStore.addPartition(ptn2); ColumnStatistics stats = new ColumnStatistics(); @@ -851,10 +879,10 @@ public class TestCachedStore { List<String> aggrPartVals = new ArrayList<>(); aggrPartVals.add("1"); aggrPartVals.add("2"); - AggrStats aggrStats = cachedStore.get_aggr_stats_for(dbName, tblName, aggrPartVals, colNames); + AggrStats aggrStats = cachedStore.get_aggr_stats_for(DEFAULT_CATALOG_NAME, dbName, tblName, aggrPartVals, colNames); Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumNulls(), 100); Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumDVs(), 5); - aggrStats = cachedStore.get_aggr_stats_for(dbName, tblName, aggrPartVals, colNames); + aggrStats = cachedStore.get_aggr_stats_for(DEFAULT_CATALOG_NAME, dbName, tblName, aggrPartVals, colNames); Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumNulls(), 100); Assert.assertEquals(aggrStats.getColStats().get(0).getStatsData().getLongStats().getNumDVs(), 5); } @@ -885,7 +913,7 @@ public class TestCachedStore { } executor.invokeAll(tasks); for (String dbName : dbNames) { - Database db = sharedCache.getDatabaseFromCache(dbName); + Database db = sharedCache.getDatabaseFromCache(DEFAULT_CATALOG_NAME, dbName); Assert.assertNotNull(db); Assert.assertEquals(dbName, db.getName()); } @@ -906,7 +934,7 @@ public class TestCachedStore { Callable<Object> c = new Callable<Object>() { public Object call() { Table tbl = createTestTbl(dbNames.get(0), tblName, "user1", cols, ptnCols); - sharedCache.addTableToCache(dbNames.get(0), tblName, tbl); + sharedCache.addTableToCache(DEFAULT_CATALOG_NAME, dbNames.get(0), tblName, tbl); return null; } }; @@ -914,7 +942,7 @@ public class TestCachedStore { } executor.invokeAll(tasks); for (String tblName : tblNames) { - Table tbl = sharedCache.getTableFromCache(dbNames.get(0), tblName); + Table tbl = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME, dbNames.get(0), tblName); Assert.assertNotNull(tbl); Assert.assertEquals(tblName, tbl.getTableName()); } @@ -923,14 +951,14 @@ public class TestCachedStore { List<String> ptnVals = new ArrayList<String>(Arrays.asList("aaa", "bbb", "ccc", "ddd", "eee")); tasks.clear(); for (String tblName : tblNames) { - Table tbl = sharedCache.getTableFromCache(dbNames.get(0), tblName); + Table tbl = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME, dbNames.get(0), tblName); for (String ptnVal : ptnVals) { Map<String, String> partParams = new HashMap<String, String>(); Callable<Object> c = new Callable<Object>() { public Object call() { Partition ptn = new Partition(Arrays.asList(ptnVal), dbNames.get(0), tblName, 0, 0, tbl.getSd(), partParams); - sharedCache.addPartitionToCache(dbNames.get(0), tblName, ptn); + sharedCache.addPartitionToCache(DEFAULT_CATALOG_NAME, dbNames.get(0), tblName, ptn); return null; } }; @@ -940,7 +968,7 @@ public class TestCachedStore { executor.invokeAll(tasks); for (String tblName : tblNames) { for (String ptnVal : ptnVals) { - Partition ptn = sharedCache.getPartitionFromCache(dbNames.get(0), tblName, Arrays.asList(ptnVal)); + Partition ptn = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME, dbNames.get(0), tblName, Arrays.asList(ptnVal)); Assert.assertNotNull(ptn); Assert.assertEquals(tblName, ptn.getTableName()); Assert.assertEquals(tblName, ptn.getTableName()); @@ -957,7 +985,7 @@ public class TestCachedStore { for (String ptnVal : ptnVals) { Callable<Object> c = new Callable<Object>() { public Object call() { - sharedCache.removePartitionFromCache(dbNames.get(0), tblName, Arrays.asList(ptnVal)); + sharedCache.removePartitionFromCache(DEFAULT_CATALOG_NAME, dbNames.get(0), tblName, Arrays.asList(ptnVal)); return null; } }; @@ -965,14 +993,14 @@ public class TestCachedStore { } } for (String tblName : addPtnTblNames) { - Table tbl = sharedCache.getTableFromCache(dbNames.get(0), tblName); + Table tbl = sharedCache.getTableFromCache(DEFAULT_CATALOG_NAME, dbNames.get(0), tblName); for (String ptnVal : newPtnVals) { Map<String, String> partParams = new HashMap<String, String>(); Callable<Object> c = new Callable<Object>() { public Object call() { Partition ptn = new Partition(Arrays.asList(ptnVal), dbNames.get(0), tblName, 0, 0, tbl.getSd(), partParams); - sharedCache.addPartitionToCache(dbNames.get(0), tblName, ptn); + sharedCache.addPartitionToCache(DEFAULT_CATALOG_NAME, dbNames.get(0), tblName, ptn); return null; } }; @@ -982,7 +1010,7 @@ public class TestCachedStore { executor.invokeAll(tasks); for (String tblName : addPtnTblNames) { for (String ptnVal : newPtnVals) { - Partition ptn = sharedCache.getPartitionFromCache(dbNames.get(0), tblName, Arrays.asList(ptnVal)); + Partition ptn = sharedCache.getPartitionFromCache(DEFAULT_CATALOG_NAME, dbNames.get(0), tblName, Arrays.asList(ptnVal)); Assert.assertNotNull(ptn); Assert.assertEquals(tblName, ptn.getTableName()); Assert.assertEquals(tblName, ptn.getTableName()); @@ -990,7 +1018,7 @@ public class TestCachedStore { } } for (String tblName : dropPtnTblNames) { - List<Partition> ptns = sharedCache.listCachedPartitions(dbNames.get(0), tblName, 100); + List<Partition> ptns = sharedCache.listCachedPartitions(DEFAULT_CATALOG_NAME, dbNames.get(0), tblName, 100); Assert.assertEquals(0, ptns.size()); } sharedCache.getDatabaseCache().clear(); @@ -1005,6 +1033,7 @@ public class TestCachedStore { Database db = new Database(dbName, dbDescription, dbLocation, dbParams); db.setOwnerName(dbOwner); db.setOwnerType(PrincipalType.USER); + db.setCatalogName(DEFAULT_CATALOG_NAME); return db; } @@ -1019,6 +1048,7 @@ public class TestCachedStore { sd.setStoredAsSubDirectories(false); Table tbl = new Table(tblName, dbName, tblOwner, 0, 0, 0, sd, ptnCols, tblParams, null, null, TableType.MANAGED_TABLE.toString()); + tbl.setCatName(DEFAULT_CATALOG_NAME); return tbl; }
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCatalogCaching.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCatalogCaching.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCatalogCaching.java new file mode 100644 index 0000000..423dce8 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCatalogCaching.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore.cache; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.ObjectStore; +import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.hadoop.hive.metastore.api.Catalog; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.util.Comparator; +import java.util.List; + +/** + * Tests that catalogs are properly cached. + */ +@Category(MetastoreCheckinTest.class) +public class TestCatalogCaching { + private static final String CAT1_NAME = "cat1"; + private static final String CAT2_NAME = "cat2"; + + private ObjectStore objectStore; + private Configuration conf; + private CachedStore cachedStore; + + @Before + public void createObjectStore() throws MetaException, InvalidOperationException { + conf = MetastoreConf.newMetastoreConf(); + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true); + MetaStoreTestUtils.setConfForStandloneMode(conf); + objectStore = new ObjectStore(); + objectStore.setConf(conf); + + // Create three catalogs + HiveMetaStore.HMSHandler.createDefaultCatalog(objectStore, new Warehouse(conf)); + + Catalog cat1 = new CatalogBuilder() + .setName(CAT1_NAME) + .setLocation("/tmp/cat1") + .build(); + objectStore.createCatalog(cat1); + Catalog cat2 = new CatalogBuilder() + .setName(CAT2_NAME) + .setLocation("/tmp/cat2") + .build(); + objectStore.createCatalog(cat2); + } + + @After + public void clearCatalogCache() throws MetaException, NoSuchObjectException { + List<String> catalogs = objectStore.getCatalogs(); + for (String catalog : catalogs) objectStore.dropCatalog(catalog); + } + + @Test + public void defaultHiveOnly() throws Exception { + // By default just the Hive catalog should be cached. + cachedStore = new CachedStore(); + cachedStore.setConf(conf); + CachedStore.stopCacheUpdateService(1); + cachedStore.resetCatalogCache(); + + CachedStore.prewarm(objectStore); + + // Only the hive catalog should be cached + List<String> cachedCatalogs = cachedStore.getCatalogs(); + Assert.assertEquals(1, cachedCatalogs.size()); + Assert.assertEquals(Warehouse.DEFAULT_CATALOG_NAME, cachedCatalogs.get(0)); + } + + @Test + public void cacheAll() throws Exception { + // Set the config value to empty string, which should result in all catalogs being cached. + Configuration newConf = new Configuration(conf); + MetastoreConf.setVar(newConf, MetastoreConf.ConfVars.CATALOGS_TO_CACHE, ""); + cachedStore = new CachedStore(); + cachedStore.setConf(newConf); + CachedStore.stopCacheUpdateService(1); + objectStore.setConf(newConf); // have to override it with the new conf since this is where + // prewarm gets the conf object + cachedStore.resetCatalogCache(); + + CachedStore.prewarm(objectStore); + + // All the catalogs should be cached + List<String> cachedCatalogs = cachedStore.getCatalogs(); + Assert.assertEquals(3, cachedCatalogs.size()); + cachedCatalogs.sort(Comparator.naturalOrder()); + Assert.assertEquals(CAT1_NAME, cachedCatalogs.get(0)); + Assert.assertEquals(CAT2_NAME, cachedCatalogs.get(1)); + Assert.assertEquals(Warehouse.DEFAULT_CATALOG_NAME, cachedCatalogs.get(2)); + } + + @Test + public void cacheSome() throws Exception { + // Set the config value to 2 catalogs other than hive + Configuration newConf = new Configuration(conf); + MetastoreConf.setVar(newConf, MetastoreConf.ConfVars.CATALOGS_TO_CACHE, CAT1_NAME + "," + CAT2_NAME); + cachedStore = new CachedStore(); + cachedStore.setConf(newConf); + CachedStore.stopCacheUpdateService(1); + objectStore.setConf(newConf); // have to override it with the new conf since this is where + // prewarm gets the conf object + cachedStore.resetCatalogCache(); + + CachedStore.prewarm(objectStore); + + // All the catalogs should be cached + List<String> cachedCatalogs = cachedStore.getCatalogs(); + Assert.assertEquals(2, cachedCatalogs.size()); + cachedCatalogs.sort(Comparator.naturalOrder()); + Assert.assertEquals(CAT1_NAME, cachedCatalogs.get(0)); + Assert.assertEquals(CAT2_NAME, cachedCatalogs.get(1)); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java index 84c187b..1a57df2 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/MetaStoreFactoryForTests.java @@ -87,6 +87,7 @@ public final class MetaStoreFactoryForTests { // Create Embedded MetaStore conf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db1;create=true"); + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.TRY_DIRECT_SQL, false); AbstractMetaStoreService embedded = new MiniHMS.Builder() .setConf(conf) @@ -97,6 +98,7 @@ public final class MetaStoreFactoryForTests { // Create Remote MetaStore conf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:memory:${test.tmp.dir}/junit_metastore_db2;create=true"); + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.TRY_DIRECT_SQL, true); AbstractMetaStoreService remote = new MiniHMS.Builder() .setConf(conf) http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java index 4d9cb1b..8555eee 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitions.java @@ -19,12 +19,16 @@ package org.apache.hadoop.hive.metastore.client; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; @@ -34,6 +38,7 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; @@ -81,10 +86,9 @@ public class TestAddPartitions extends MetaStoreClientTest { // Clean up the database client.dropDatabase(DB_NAME, true, true, true); metaStore.cleanWarehouseDirs(); - Database db = new DatabaseBuilder(). + new DatabaseBuilder(). setName(DB_NAME). - build(); - client.createDatabase(db); + create(client, metaStore.getConf()); } @After @@ -123,6 +127,72 @@ public class TestAddPartitions extends MetaStoreClientTest { } @Test + public void addPartitionOtherCatalog() throws TException { + String catName = "add_partition_catalog"; + Catalog cat = new CatalogBuilder() + .setName(catName) + .setLocation(MetaStoreTestUtils.getTestWarehouseDir(catName)) + .build(); + client.createCatalog(cat); + + String dbName = "add_partition_database_in_other_catalog"; + Database db = new DatabaseBuilder() + .setName(dbName) + .setCatalogName(catName) + .create(client, metaStore.getConf()); + + String tableName = "table_in_other_catalog"; + Table table = new TableBuilder() + .inDb(db) + .setTableName(tableName) + .addCol("id", "int") + .addCol("name", "string") + .addPartCol("partcol", "string") + .create(client, metaStore.getConf()); + + Partition[] parts = new Partition[5]; + for (int i = 0; i < parts.length; i++) { + parts[i] = new PartitionBuilder() + .inTable(table) + .addValue("a" + i) + .build(metaStore.getConf()); + } + client.add_partition(parts[0]); + Assert.assertEquals(2, client.add_partitions(Arrays.asList(parts[1], parts[2]))); + client.add_partitions(Arrays.asList(parts), true, false); + + for (int i = 0; i < parts.length; i++) { + Partition fetched = client.getPartition(catName, dbName, tableName, + Collections.singletonList("a" + i)); + Assert.assertEquals(catName, fetched.getCatName()); + Assert.assertEquals(dbName, fetched.getDbName()); + Assert.assertEquals(tableName, fetched.getTableName()); + } + + client.dropDatabase(catName, dbName, true, true, true); + client.dropCatalog(catName); + } + + @Test(expected = InvalidObjectException.class) + public void noSuchCatalog() throws TException { + String tableName = "table_for_no_such_catalog"; + Table table = new TableBuilder() + .setTableName(tableName) + .addCol("id", "int") + .addCol("name", "string") + .addPartCol("partcol", "string") + .create(client, metaStore.getConf()); + + Partition part = new PartitionBuilder() + .inTable(table) + .addValue("a") + .build(metaStore.getConf()); + // Explicitly mis-set the catalog name + part.setCatName("nosuch"); + client.add_partition(part); + } + + @Test public void testAddPartitionWithDefaultAttributes() throws Exception { Table table = createTable(); @@ -134,7 +204,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .setCols(getYearPartCol()) .addCol("test_id", "int", "test col id") .addCol("test_value", "string", "test col value") - .build(); + .build(metaStore.getConf()); client.add_partition(partition); @@ -270,7 +340,7 @@ public class TestAddPartitions extends MetaStoreClientTest { createTable(); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); - partition.getSd().setCols(new ArrayList<FieldSchema>()); + partition.getSd().setCols(new ArrayList<>()); client.add_partition(partition); // TODO: Not sure that this is the correct behavior. It doesn't make sense to create the @@ -372,8 +442,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .addCol("test_value", DEFAULT_COL_TYPE, "test col value") .addPartCol(YEAR_COL_NAME, DEFAULT_COL_TYPE) .setLocation(null) - .build(); - client.createTable(table); + .create(client, metaStore.getConf()); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); client.add_partition(partition); } @@ -427,8 +496,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .setTableName(TABLE_NAME) .addCol("test_id", "int", "test col id") .addCol("test_value", "string", "test col value") - .build(); - client.createTable(origTable); + .create(client, metaStore.getConf()); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); client.add_partition(partition); } @@ -442,7 +510,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .setTableName(TABLE_NAME) .addValue(DEFAULT_YEAR_VALUE) .setLocation(metaStore.getWarehouseRoot() + "/addparttest") - .build(); + .build(metaStore.getConf()); client.add_partition(partition); } @@ -455,7 +523,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .setTableName(TABLE_NAME) .addValue("1000") .addCol("time", "int") - .build(); + .build(metaStore.getConf()); client.add_partition(partition); Partition part = client.getPartition(DB_NAME, TABLE_NAME, "year=1000"); @@ -474,7 +542,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .setTableName(TABLE_NAME) .addCol(YEAR_COL_NAME, DEFAULT_COL_TYPE) .setLocation(metaStore.getWarehouseRoot() + "/addparttest") - .build(); + .build(metaStore.getConf()); client.add_partition(partition); } @@ -588,7 +656,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .setCols(getYearPartCol()) .addCol("test_id", "int", "test col id") .addCol("test_value", "string", "test col value") - .build(); + .build(metaStore.getConf()); client.add_partitions(Lists.newArrayList(partition)); @@ -622,7 +690,7 @@ public class TestAddPartitions extends MetaStoreClientTest { @Test public void testAddPartitionsEmptyList() throws Exception { - client.add_partitions(new ArrayList<Partition>()); + client.add_partitions(new ArrayList<>()); } @Test(expected = MetaException.class) @@ -873,7 +941,7 @@ public class TestAddPartitions extends MetaStoreClientTest { createTable(); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); - partition.getSd().setCols(new ArrayList<FieldSchema>()); + partition.getSd().setCols(new ArrayList<>()); client.add_partitions(Lists.newArrayList(partition)); // TODO: Not sure that this is the correct behavior. It doesn't make sense to create the @@ -976,8 +1044,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .addCol("test_value", "string", "test col value") .addPartCol(YEAR_COL_NAME, DEFAULT_COL_TYPE) .setLocation(null) - .build(); - client.createTable(table); + .create(client, metaStore.getConf()); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); List<Partition> partitions = Lists.newArrayList(partition); client.add_partitions(partitions); @@ -1044,7 +1111,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .setTableName(TABLE_NAME) .addCol(YEAR_COL_NAME, DEFAULT_COL_TYPE) .setLocation(metaStore.getWarehouseRoot() + "/addparttest") - .build(); + .build(metaStore.getConf()); List<Partition> partitions = new ArrayList<>(); partitions.add(partition); client.add_partitions(partitions); @@ -1160,7 +1227,7 @@ public class TestAddPartitions extends MetaStoreClientTest { public void testAddPartsEmptyList() throws Exception { List<Partition> addedPartitions = - client.add_partitions(new ArrayList<Partition>(), false, true); + client.add_partitions(new ArrayList<>(), false, true); Assert.assertNotNull(addedPartitions); Assert.assertTrue(addedPartitions.isEmpty()); } @@ -1276,8 +1343,7 @@ public class TestAddPartitions extends MetaStoreClientTest { // Helper methods private void createDB(String dbName) throws TException { - Database db = new DatabaseBuilder().setName(dbName).build(); - client.createDatabase(db); + new DatabaseBuilder().setName(dbName).create(client, metaStore.getConf()); } private Table createTable() throws Exception { @@ -1302,13 +1368,12 @@ public class TestAddPartitions extends MetaStoreClientTest { .setStoredAsSubDirectories(false) .addSerdeParam("partTestSerdeParamKey", "partTestSerdeParamValue") .setLocation(location) - .build(); - client.createTable(table); + .create(client, metaStore.getConf()); return client.getTable(dbName, tableName); } private void createExternalTable(String tableName, String location) throws Exception { - Table table = new TableBuilder() + new TableBuilder() .setDbName(DB_NAME) .setTableName(tableName) .addCol("test_id", "int", "test col id") @@ -1316,8 +1381,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .addPartCol(YEAR_COL_NAME, DEFAULT_COL_TYPE) .addTableParam("EXTERNAL", "TRUE") .setLocation(location) - .build(); - client.createTable(table); + .create(client, metaStore.getConf()); } private Partition buildPartition(String dbName, String tableName, String value) @@ -1337,7 +1401,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .addCol("test_value", "string", "test col value") .addPartParam(DEFAULT_PARAM_KEY, DEFAULT_PARAM_VALUE) .setLocation(location) - .build(); + .build(metaStore.getConf()); return partition; } @@ -1357,7 +1421,7 @@ public class TestAddPartitions extends MetaStoreClientTest { .setLastAccessTime(123456) .addCol("test_id", "int", "test col id") .addCol("test_value", "string", "test col value") - .build(); + .build(metaStore.getConf()); return partition; } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java index 1122057..b32954f 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAddPartitionsFromPartSpec.java @@ -88,10 +88,9 @@ public class TestAddPartitionsFromPartSpec extends MetaStoreClientTest { // Clean up the database client.dropDatabase(DB_NAME, true, true, true); metaStore.cleanWarehouseDirs(); - Database db = new DatabaseBuilder(). + new DatabaseBuilder(). setName(DB_NAME). - build(); - client.createDatabase(db); + create(client, metaStore.getConf()); } @After @@ -169,6 +168,8 @@ public class TestAddPartitionsFromPartSpec extends MetaStoreClientTest { verifyPartitionSharedSD(table, "year=2005/month=may", Lists.newArrayList("2005", "may"), 4); } + // TODO add tests for partitions in other catalogs + @Test(expected = NullPointerException.class) public void testAddPartitionSpecNullSpec() throws Exception { @@ -679,8 +680,7 @@ public class TestAddPartitionsFromPartSpec extends MetaStoreClientTest { .addCol("test_value", DEFAULT_COL_TYPE, "test col value") .addPartCol(YEAR_COL_NAME, DEFAULT_COL_TYPE) .setLocation(null) - .build(); - client.createTable(table); + .create(client, metaStore.getConf()); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE); PartitionSpecProxy partitionSpecProxy = @@ -714,7 +714,7 @@ public class TestAddPartitionsFromPartSpec extends MetaStoreClientTest { .setTableName(TABLE_NAME) .addCol(YEAR_COL_NAME, DEFAULT_COL_TYPE) .setLocation(metaStore.getWarehouseRoot() + "/addpartspectest") - .build(); + .build(metaStore.getConf()); PartitionSpecProxy partitionSpecProxy = buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); @@ -821,8 +821,7 @@ public class TestAddPartitionsFromPartSpec extends MetaStoreClientTest { // Helper methods private void createDB(String dbName) throws TException { - Database db = new DatabaseBuilder().setName(dbName).build(); - client.createDatabase(db); + Database db = new DatabaseBuilder().setName(dbName).create(client, metaStore.getConf()); } private Table createTable() throws Exception { @@ -844,8 +843,7 @@ public class TestAddPartitionsFromPartSpec extends MetaStoreClientTest { .setStoredAsSubDirectories(false) .addSerdeParam("partTestSerdeParamKey", "partTestSerdeParamValue") .setLocation(location) - .build(); - client.createTable(table); + .create(client, metaStore.getConf()); return client.getTable(dbName, tableName); } @@ -866,7 +864,7 @@ public class TestAddPartitionsFromPartSpec extends MetaStoreClientTest { .addCol("test_value", "string", "test col value") .addPartParam(DEFAULT_PARAM_KEY, DEFAULT_PARAM_VALUE) .setLocation(location) - .build(); + .build(metaStore.getConf()); return partition; } @@ -886,7 +884,7 @@ public class TestAddPartitionsFromPartSpec extends MetaStoreClientTest { .setLastAccessTime(DEFAULT_CREATE_TIME) .addCol("test_id", "int", "test col id") .addCol("test_value", "string", "test col value") - .build(); + .build(metaStore.getConf()); return partition; } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java index 747f66d..770da1a 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java @@ -19,11 +19,15 @@ package org.apache.hadoop.hive.metastore.client; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -31,6 +35,7 @@ import org.apache.hadoop.hive.metastore.api.InvalidOperationException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; @@ -41,6 +46,8 @@ import org.apache.thrift.transport.TTransportException; import com.google.common.collect.Lists; import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -48,6 +55,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import static java.util.stream.Collectors.joining; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; @@ -60,7 +68,7 @@ import static org.junit.Assert.fail; @RunWith(Parameterized.class) @Category(MetastoreCheckinTest.class) public class TestAlterPartitions extends MetaStoreClientTest { - public static final int NEW_CREATE_TIME = 123456789; + private static final int NEW_CREATE_TIME = 123456789; private AbstractMetaStoreService metaStore; private IMetaStoreClient client; @@ -95,13 +103,12 @@ public class TestAlterPartitions extends MetaStoreClientTest { } private void createDB(String dbName) throws TException { - Database db = new DatabaseBuilder(). + new DatabaseBuilder(). setName(dbName). - build(); - client.createDatabase(db); + create(client, metaStore.getConf()); } - private static Table createTestTable(IMetaStoreClient client, String dbName, String tableName, + private Table createTestTable(IMetaStoreClient client, String dbName, String tableName, List<String> partCols, boolean setPartitionLevelPrivilages) throws Exception { TableBuilder builder = new TableBuilder() @@ -111,7 +118,7 @@ public class TestAlterPartitions extends MetaStoreClientTest { .addCol("name", "string"); partCols.forEach(col -> builder.addPartCol(col, "string")); - Table table = builder.build(); + Table table = builder.build(metaStore.getConf()); if (setPartitionLevelPrivilages) { table.putToParameters("PARTITION_LEVEL_PRIVILEGE", "true"); @@ -121,14 +128,14 @@ public class TestAlterPartitions extends MetaStoreClientTest { return table; } - private static void addPartition(IMetaStoreClient client, Table table, List<String> values) + private void addPartition(IMetaStoreClient client, Table table, List<String> values) throws TException { - PartitionBuilder partitionBuilder = new PartitionBuilder().fromTable(table); + PartitionBuilder partitionBuilder = new PartitionBuilder().inTable(table); values.forEach(val -> partitionBuilder.addValue(val)); - client.add_partition(partitionBuilder.build()); + client.add_partition(partitionBuilder.build(metaStore.getConf())); } - private static List<List<String>> createTable4PartColsParts(IMetaStoreClient client) throws + private List<List<String>> createTable4PartColsParts(IMetaStoreClient client) throws Exception { Table t = createTestTable(client, DB_NAME, TABLE_NAME, PARTCOL_SCHEMA, false); List<List<String>> testValues = Lists.newArrayList( @@ -197,7 +204,6 @@ public class TestAlterPartitions extends MetaStoreClientTest { /** * Testing alter_partition(String,String,Partition) -> * alter_partition_with_environment_context(String,String,Partition,null). - * @throws Exception */ @Test public void testAlterPartition() throws Exception { @@ -217,12 +223,152 @@ public class TestAlterPartitions extends MetaStoreClientTest { } + @Test + public void otherCatalog() throws TException { + String catName = "alter_partition_catalog"; + Catalog cat = new CatalogBuilder() + .setName(catName) + .setLocation(MetaStoreTestUtils.getTestWarehouseDir(catName)) + .build(); + client.createCatalog(cat); + + String dbName = "alter_partition_database_in_other_catalog"; + Database db = new DatabaseBuilder() + .setName(dbName) + .setCatalogName(catName) + .create(client, metaStore.getConf()); + + String tableName = "table_in_other_catalog"; + Table table = new TableBuilder() + .inDb(db) + .setTableName(tableName) + .addCol("id", "int") + .addCol("name", "string") + .addPartCol("partcol", "string") + .create(client, metaStore.getConf()); + + Partition[] parts = new Partition[5]; + for (int i = 0; i < 5; i++) { + parts[i] = new PartitionBuilder() + .inTable(table) + .addValue("a" + i) + .setLocation(MetaStoreTestUtils.getTestWarehouseDir("b" + i)) + .build(metaStore.getConf()); + } + client.add_partitions(Arrays.asList(parts)); + + Partition newPart = + client.getPartition(catName, dbName, tableName, Collections.singletonList("a0")); + newPart.getParameters().put("test_key", "test_value"); + client.alter_partition(catName, dbName, tableName, newPart); + + Partition fetched = + client.getPartition(catName, dbName, tableName, Collections.singletonList("a0")); + Assert.assertEquals(catName, fetched.getCatName()); + Assert.assertEquals("test_value", fetched.getParameters().get("test_key")); + + newPart = + client.getPartition(catName, dbName, tableName, Collections.singletonList("a1")); + newPart.setLastAccessTime(3); + Partition newPart1 = + client.getPartition(catName, dbName, tableName, Collections.singletonList("a2")); + newPart1.getSd().setLocation(MetaStoreTestUtils.getTestWarehouseDir("somewhere")); + client.alter_partitions(catName, dbName, tableName, Arrays.asList(newPart, newPart1)); + fetched = + client.getPartition(catName, dbName, tableName, Collections.singletonList("a1")); + Assert.assertEquals(catName, fetched.getCatName()); + Assert.assertEquals(3L, fetched.getLastAccessTime()); + fetched = + client.getPartition(catName, dbName, tableName, Collections.singletonList("a2")); + Assert.assertEquals(catName, fetched.getCatName()); + Assert.assertTrue(fetched.getSd().getLocation().contains("somewhere")); + + newPart = + client.getPartition(catName, dbName, tableName, Collections.singletonList("a4")); + newPart.getParameters().put("test_key", "test_value"); + EnvironmentContext ec = new EnvironmentContext(); + ec.setProperties(Collections.singletonMap("a", "b")); + client.alter_partition(catName, dbName, tableName, newPart, ec); + fetched = + client.getPartition(catName, dbName, tableName, Collections.singletonList("a4")); + Assert.assertEquals(catName, fetched.getCatName()); + Assert.assertEquals("test_value", fetched.getParameters().get("test_key")); + + + client.dropDatabase(catName, dbName, true, true, true); + client.dropCatalog(catName); + } + + @SuppressWarnings("deprecation") + @Test + public void deprecatedCalls() throws TException { + String tableName = "deprecated_table"; + Table table = new TableBuilder() + .setTableName(tableName) + .addCol("id", "int") + .addCol("name", "string") + .addPartCol("partcol", "string") + .create(client, metaStore.getConf()); + + Partition[] parts = new Partition[5]; + for (int i = 0; i < 5; i++) { + parts[i] = new PartitionBuilder() + .inTable(table) + .addValue("a" + i) + .setLocation(MetaStoreTestUtils.getTestWarehouseDir("a" + i)) + .build(metaStore.getConf()); + } + client.add_partitions(Arrays.asList(parts)); + + Partition newPart = + client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a0")); + newPart.getParameters().put("test_key", "test_value"); + client.alter_partition(DEFAULT_DATABASE_NAME, tableName, newPart); + + Partition fetched = + client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a0")); + Assert.assertEquals("test_value", fetched.getParameters().get("test_key")); + + newPart = + client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a1")); + newPart.setLastAccessTime(3); + Partition newPart1 = + client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a2")); + newPart1.getSd().setLocation("somewhere"); + client.alter_partitions(DEFAULT_DATABASE_NAME, tableName, Arrays.asList(newPart, newPart1)); + fetched = + client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a1")); + Assert.assertEquals(3L, fetched.getLastAccessTime()); + fetched = + client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a2")); + Assert.assertTrue(fetched.getSd().getLocation().contains("somewhere")); + + newPart = + client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a3")); + newPart.setValues(Collections.singletonList("b3")); + client.renamePartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a3"), newPart); + fetched = + client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("b3")); + Assert.assertEquals(1, fetched.getValuesSize()); + Assert.assertEquals("b3", fetched.getValues().get(0)); + + newPart = + client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a4")); + newPart.getParameters().put("test_key", "test_value"); + EnvironmentContext ec = new EnvironmentContext(); + ec.setProperties(Collections.singletonMap("a", "b")); + client.alter_partition(DEFAULT_DATABASE_NAME, tableName, newPart, ec); + fetched = + client.getPartition(DEFAULT_DATABASE_NAME, tableName, Collections.singletonList("a4")); + Assert.assertEquals("test_value", fetched.getParameters().get("test_key")); + } + @Test(expected = InvalidOperationException.class) public void testAlterPartitionUnknownPartition() throws Exception { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).addValue("1111").addValue("11").addValue("11").build(); + Partition part = builder.inTable(t).addValue("1111").addValue("11").addValue("11").build(metaStore.getConf()); client.alter_partition(DB_NAME, TABLE_NAME, part); } @@ -231,7 +377,7 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).addValue("2017").build(); + Partition part = builder.inTable(t).addValue("2017").build(metaStore.getConf()); client.alter_partition(DB_NAME, TABLE_NAME, part); } @@ -240,11 +386,18 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).build(); + Partition part = builder.inTable(t).build(metaStore.getConf()); client.alter_partition(DB_NAME, TABLE_NAME, part); } @Test(expected = InvalidOperationException.class) + public void testAlterPartitionBogusCatalogName() throws Exception { + createTable4PartColsParts(client); + List<Partition> partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1); + client.alter_partition("nosuch", DB_NAME, TABLE_NAME, partitions.get(3)); + } + + @Test(expected = InvalidOperationException.class) public void testAlterPartitionNoDbName() throws Exception { createTable4PartColsParts(client); List<Partition> partitions = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1); @@ -315,7 +468,6 @@ public class TestAlterPartitions extends MetaStoreClientTest { /** * Testing alter_partition(String,String,Partition,EnvironmentContext) -> * alter_partition_with_environment_context(String,String,Partition,EnvironmentContext). - * @throws Exception */ @Test public void testAlterPartitionWithEnvironmentCtx() throws Exception { @@ -349,7 +501,7 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).addValue("1111").addValue("11").addValue("11").build(); + Partition part = builder.inTable(t).addValue("1111").addValue("11").addValue("11").build(metaStore.getConf()); client.alter_partition(DB_NAME, TABLE_NAME, part, new EnvironmentContext()); } @@ -358,7 +510,7 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).addValue("2017").build(); + Partition part = builder.inTable(t).addValue("2017").build(metaStore.getConf()); client.alter_partition(DB_NAME, TABLE_NAME, part, new EnvironmentContext()); } @@ -367,7 +519,7 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).build(); + Partition part = builder.inTable(t).build(metaStore.getConf()); client.alter_partition(DB_NAME, TABLE_NAME, part, new EnvironmentContext()); } @@ -444,7 +596,6 @@ public class TestAlterPartitions extends MetaStoreClientTest { * Testing * alter_partitions(String,String,List(Partition)) -> * alter_partitions_with_environment_context(String,String,List(Partition),null). - * @throws Exception */ @Test public void testAlterPartitions() throws Exception { @@ -478,7 +629,7 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).addValue("1111").addValue("11").addValue("11").build(); + Partition part = builder.inTable(t).addValue("1111").addValue("11").addValue("11").build(metaStore.getConf()); part1 = client.listPartitions(DB_NAME, TABLE_NAME, (short) -1).get(0); makeTestChangesOnPartition(part1); client.alter_partitions(DB_NAME, TABLE_NAME, Lists.newArrayList(part, part1)); @@ -494,7 +645,7 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).addValue("2017").build(); + Partition part = builder.inTable(t).addValue("2017").build(metaStore.getConf()); Partition part1 = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); client.alter_partitions(DB_NAME, TABLE_NAME, Lists.newArrayList(part, part1)); } @@ -504,12 +655,19 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).build(); + Partition part = builder.inTable(t).build(metaStore.getConf()); Partition part1 = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); client.alter_partitions(DB_NAME, TABLE_NAME, Lists.newArrayList(part, part1)); } @Test(expected = InvalidOperationException.class) + public void testAlterPartitionsBogusCatalogName() throws Exception { + createTable4PartColsParts(client); + Partition part = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); + client.alter_partitions("nosuch", DB_NAME, TABLE_NAME, Lists.newArrayList(part)); + } + + @Test(expected = InvalidOperationException.class) public void testAlterPartitionsNoDbName() throws Exception { createTable4PartColsParts(client); Partition part = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); @@ -596,7 +754,6 @@ public class TestAlterPartitions extends MetaStoreClientTest { * Testing * alter_partitions(String,String,List(Partition),EnvironmentContext) -> * alter_partitions_with_environment_context(String,String,List(Partition),EnvironmentContext). - * @throws Exception */ @Test public void testAlterPartitionsWithEnvironmentCtx() throws Exception { @@ -642,7 +799,7 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).addValue("1111").addValue("11").addValue("11").build(); + Partition part = builder.inTable(t).addValue("1111").addValue("11").addValue("11").build(metaStore.getConf()); Partition part1 = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); client.alter_partitions(DB_NAME, TABLE_NAME, Lists.newArrayList(part, part1), new EnvironmentContext()); @@ -653,7 +810,7 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).addValue("2017").build(); + Partition part = builder.inTable(t).addValue("2017").build(metaStore.getConf()); Partition part1 = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); client.alter_partitions(DB_NAME, TABLE_NAME, Lists.newArrayList(part, part1), new EnvironmentContext()); @@ -664,13 +821,20 @@ public class TestAlterPartitions extends MetaStoreClientTest { createTable4PartColsParts(client); Table t = client.getTable(DB_NAME, TABLE_NAME); PartitionBuilder builder = new PartitionBuilder(); - Partition part = builder.fromTable(t).build(); + Partition part = builder.inTable(t).build(metaStore.getConf()); Partition part1 = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); client.alter_partitions(DB_NAME, TABLE_NAME, Lists.newArrayList(part, part1), new EnvironmentContext()); } @Test(expected = InvalidOperationException.class) + public void testAlterPartitionsWithEnvironmentCtxBogusCatalogName() throws Exception { + createTable4PartColsParts(client); + Partition part = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); + client.alter_partitions("nosuch", DB_NAME, TABLE_NAME, Lists.newArrayList(part), new EnvironmentContext()); + } + + @Test(expected = InvalidOperationException.class) public void testAlterPartitionsWithEnvironmentCtxNoDbName() throws Exception { createTable4PartColsParts(client); Partition part = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); @@ -757,7 +921,6 @@ public class TestAlterPartitions extends MetaStoreClientTest { * Testing * renamePartition(String,String,List(String),Partition) -> * renamePartition(String,String,List(String),Partition). - * @throws Exception */ @Test public void testRenamePartition() throws Exception { @@ -870,6 +1033,16 @@ public class TestAlterPartitions extends MetaStoreClientTest { } @Test(expected = InvalidOperationException.class) + public void testRenamePartitionBogusCatalogName() throws Exception { + List<List<String>> oldValues = createTable4PartColsParts(client); + List<Partition> oldParts = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1); + + Partition partToRename = oldParts.get(3); + partToRename.setValues(Lists.newArrayList("2018", "01", "16")); + client.renamePartition("nosuch", DB_NAME, TABLE_NAME, oldValues.get(3), partToRename); + } + + @Test(expected = InvalidOperationException.class) public void testRenamePartitionNoDbName() throws Exception { List<List<String>> oldValues = createTable4PartColsParts(client); List<Partition> oldParts = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1); http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java index 79d0953..75b26f2 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAppendPartitions.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.metastore.client; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,9 +28,11 @@ import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; @@ -37,10 +40,13 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; import org.apache.hadoop.hive.metastore.minihms.AbstractMetaStoreService; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -78,10 +84,9 @@ public class TestAppendPartitions extends MetaStoreClientTest { // Clean up the database client.dropDatabase(DB_NAME, true, true, true); metaStore.cleanWarehouseDirs(); - Database db = new DatabaseBuilder() + new DatabaseBuilder() .setName(DB_NAME) - .build(); - client.createDatabase(db); + .create(client, metaStore.getConf()); tableWithPartitions = createTableWithPartitions(); externalTable = createExternalTable(); @@ -221,18 +226,22 @@ public class TestAppendPartitions extends MetaStoreClientTest { client.appendPartition(tableWithPartitions.getDbName(), null, partitionValues); } - @Test(expected = MetaException.class) + @Test(expected = InvalidObjectException.class) public void testAppendPartitionEmptyPartValues() throws Exception { Table table = tableWithPartitions; - client.appendPartition(table.getDbName(), table.getTableName(), new ArrayList<String>()); + client.appendPartition(table.getDbName(), table.getTableName(), new ArrayList<>()); } - @Test(expected = MetaException.class) + @Test public void testAppendPartitionNullPartValues() throws Exception { - - Table table = tableWithPartitions; - client.appendPartition(table.getDbName(), table.getTableName(), (List<String>) null); + try { + Table table = tableWithPartitions; + client.appendPartition(table.getDbName(), table.getTableName(), (List<String>) null); + Assert.fail("Exception should have been thrown."); + } catch (TTransportException | InvalidObjectException e) { + // TODO: NPE should not be thrown + } } @Test @@ -436,6 +445,57 @@ public class TestAppendPartitions extends MetaStoreClientTest { client.appendPartition(table.getDbName(), table.getTableName(), partitionName); } + @Test + public void otherCatalog() throws TException { + String catName = "append_partition_catalog"; + Catalog cat = new CatalogBuilder() + .setName(catName) + .setLocation(MetaStoreTestUtils.getTestWarehouseDir(catName)) + .build(); + client.createCatalog(cat); + + String dbName = "append_partition_database_in_other_catalog"; + Database db = new DatabaseBuilder() + .setName(dbName) + .setCatalogName(catName) + .create(client, metaStore.getConf()); + + String tableName = "table_in_other_catalog"; + new TableBuilder() + .inDb(db) + .setTableName(tableName) + .addCol("id", "int") + .addCol("name", "string") + .addPartCol("partcol", "string") + .create(client, metaStore.getConf()); + + Partition created = + client.appendPartition(catName, dbName, tableName, Collections.singletonList("a1")); + Assert.assertEquals(1, created.getValuesSize()); + Assert.assertEquals("a1", created.getValues().get(0)); + Partition fetched = + client.getPartition(catName, dbName, tableName, Collections.singletonList("a1")); + Assert.assertEquals(created, fetched); + + created = client.appendPartition(catName, dbName, tableName, "partcol=a2"); + Assert.assertEquals(1, created.getValuesSize()); + Assert.assertEquals("a2", created.getValues().get(0)); + fetched = client.getPartition(catName, dbName, tableName, Collections.singletonList("a2")); + Assert.assertEquals(created, fetched); + } + + @Test(expected = InvalidObjectException.class) + public void testAppendPartitionBogusCatalog() throws Exception { + client.appendPartition("nosuch", DB_NAME, tableWithPartitions.getTableName(), + Lists.newArrayList("2017", "may")); + } + + @Test(expected = InvalidObjectException.class) + public void testAppendPartitionByNameBogusCatalog() throws Exception { + client.appendPartition("nosuch", DB_NAME, tableWithPartitions.getTableName(), + "year=2017/month=april"); + } + // Helper methods private Table createTableWithPartitions() throws Exception { @@ -471,7 +531,7 @@ public class TestAppendPartitions extends MetaStoreClientTest { private Table createTable(String tableName, List<FieldSchema> partCols, Map<String, String> tableParams, String tableType, String location) throws Exception { - Table table = new TableBuilder() + new TableBuilder() .setDbName(DB_NAME) .setTableName(tableName) .addCol("test_id", "int", "test col id") @@ -480,17 +540,15 @@ public class TestAppendPartitions extends MetaStoreClientTest { .setTableParams(tableParams) .setType(tableType) .setLocation(location) - .build(); - client.createTable(table); + .create(client, metaStore.getConf()); return client.getTable(DB_NAME, tableName); } private void createPartition(Table table, List<String> values) throws Exception { - Partition partition = new PartitionBuilder() - .fromTable(table) + new PartitionBuilder() + .inTable(table) .setValues(values) - .build(); - client.add_partition(partition); + .addToTable(client, metaStore.getConf()); } private static List<FieldSchema> getYearAndMonthPartCols() {
