http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStoreSchemaMethods.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStoreSchemaMethods.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStoreSchemaMethods.java index 2b8fbd1..137082f 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStoreSchemaMethods.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestObjectStoreSchemaMethods.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hive.metastore.api.SchemaVersion; import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; import org.apache.hadoop.hive.metastore.api.SchemaVersionState; import org.apache.hadoop.hive.metastore.api.SerDeInfo; +import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.client.builder.ISchemaBuilder; import org.apache.hadoop.hive.metastore.client.builder.SchemaVersionBuilder; @@ -47,16 +48,19 @@ import org.junit.experimental.categories.Category; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Random; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; @Category(MetastoreCheckinTest.class) public class TestObjectStoreSchemaMethods { private RawStore objectStore; + private Configuration conf; @Before public void setUp() throws Exception { - Configuration conf = MetastoreConf.newMetastoreConf(); + conf = MetastoreConf.newMetastoreConf(); MetastoreConf.setVar(conf, MetastoreConf.ConfVars.EXPRESSION_PROXY_CLASS, DefaultPartitionExpressionProxy.class.getName()); @@ -66,8 +70,8 @@ public class TestObjectStoreSchemaMethods { @Test public void iSchema() throws TException { - String dbName = createUniqueDatabaseForTest(); - ISchema schema = objectStore.getISchema(new ISchemaName(dbName, "no.such.schema")); + Database db = createUniqueDatabaseForTest(); + ISchema schema = objectStore.getISchema(new ISchemaName(db.getCatalogName(), db.getName(), "no.such.schema")); Assert.assertNull(schema); String schemaName = "schema1"; @@ -76,7 +80,7 @@ public class TestObjectStoreSchemaMethods { schema = new ISchemaBuilder() .setSchemaType(SchemaType.AVRO) .setName(schemaName) - .setDbName(dbName) + .inDb(db) .setCompatibility(SchemaCompatibility.FORWARD) .setValidationLevel(SchemaValidation.LATEST) .setCanEvolve(false) @@ -85,7 +89,7 @@ public class TestObjectStoreSchemaMethods { .build(); objectStore.createISchema(schema); - schema = objectStore.getISchema(new ISchemaName(dbName, schemaName)); + schema = objectStore.getISchema(new ISchemaName(db.getCatalogName(), db.getName(), schemaName)); Assert.assertNotNull(schema); Assert.assertEquals(SchemaType.AVRO, schema.getSchemaType()); @@ -103,9 +107,9 @@ public class TestObjectStoreSchemaMethods { schema.setCanEvolve(true); schema.setSchemaGroup(schemaGroup); schema.setDescription(description); - objectStore.alterISchema(new ISchemaName(dbName, schemaName), schema); + objectStore.alterISchema(new ISchemaName(db.getCatalogName(), db.getName(), schemaName), schema); - schema = objectStore.getISchema(new ISchemaName(dbName, schemaName)); + schema = objectStore.getISchema(new ISchemaName(db.getCatalogName(), db.getName(), schemaName)); Assert.assertNotNull(schema); Assert.assertEquals(SchemaType.AVRO, schema.getSchemaType()); @@ -116,8 +120,8 @@ public class TestObjectStoreSchemaMethods { Assert.assertEquals(schemaGroup, schema.getSchemaGroup()); Assert.assertEquals(description, schema.getDescription()); - objectStore.dropISchema(new ISchemaName(dbName, schemaName)); - schema = objectStore.getISchema(new ISchemaName(dbName, schemaName)); + objectStore.dropISchema(new ISchemaName(db.getCatalogName(), db.getName(), schemaName)); + schema = objectStore.getISchema(new ISchemaName(db.getCatalogName(), db.getName(), schemaName)); Assert.assertNull(schema); } @@ -134,16 +138,16 @@ public class TestObjectStoreSchemaMethods { @Test(expected = AlreadyExistsException.class) public void schemaAlreadyExists() throws TException { - String dbName = createUniqueDatabaseForTest(); + Database db = createUniqueDatabaseForTest(); String schemaName = "schema2"; ISchema schema = new ISchemaBuilder() .setSchemaType(SchemaType.HIVE) .setName(schemaName) - .setDbName(dbName) + .inDb(db) .build(); objectStore.createISchema(schema); - schema = objectStore.getISchema(new ISchemaName(dbName, schemaName)); + schema = objectStore.getISchema(new ISchemaName(db.getCatalogName(), db.getName(), schemaName)); Assert.assertNotNull(schema); Assert.assertEquals(SchemaType.HIVE, schema.getSchemaType()); @@ -164,12 +168,12 @@ public class TestObjectStoreSchemaMethods { .setName(schemaName) .setDescription("a new description") .build(); - objectStore.alterISchema(new ISchemaName(DEFAULT_DATABASE_NAME, schemaName), schema); + objectStore.alterISchema(new ISchemaName(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, schemaName), schema); } @Test(expected = NoSuchObjectException.class) public void dropNonExistentSchema() throws MetaException, NoSuchObjectException { - objectStore.dropISchema(new ISchemaName(DEFAULT_DATABASE_NAME, "no_such_schema")); + objectStore.dropISchema(new ISchemaName(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, "no_such_schema")); } @Test(expected = NoSuchObjectException.class) @@ -177,7 +181,6 @@ public class TestObjectStoreSchemaMethods { NoSuchObjectException, InvalidObjectException { SchemaVersion schemaVersion = new SchemaVersionBuilder() .setSchemaName("noSchemaOfThisNameExists") - .setDbName(DEFAULT_DATABASE_NAME) .setVersion(1) .addCol("a", ColumnType.STRING_TYPE_NAME) .build(); @@ -186,16 +189,16 @@ public class TestObjectStoreSchemaMethods { @Test public void addSchemaVersion() throws TException { - String dbName = createUniqueDatabaseForTest(); + Database db = createUniqueDatabaseForTest(); String schemaName = "schema37"; int version = 1; - SchemaVersion schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version)); + SchemaVersion schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(db.getCatalogName(), db.getName(), schemaName), version)); Assert.assertNull(schemaVersion); ISchema schema = new ISchemaBuilder() .setSchemaType(SchemaType.AVRO) .setName(schemaName) - .setDbName(dbName) + .inDb(db) .build(); objectStore.createISchema(schema); @@ -226,10 +229,11 @@ public class TestObjectStoreSchemaMethods { .build(); objectStore.addSchemaVersion(schemaVersion); - schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version)); + schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(db.getCatalogName(), db.getName(), schemaName), version)); Assert.assertNotNull(schemaVersion); Assert.assertEquals(schemaName, schemaVersion.getSchema().getSchemaName()); - Assert.assertEquals(dbName, schemaVersion.getSchema().getDbName()); + Assert.assertEquals(db.getName(), schemaVersion.getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), schemaVersion.getSchema().getCatName()); Assert.assertEquals(version, schemaVersion.getVersion()); Assert.assertEquals(creationTime, schemaVersion.getCreatedAt()); Assert.assertEquals(SchemaVersionState.INITIATED, schemaVersion.getState()); @@ -249,21 +253,21 @@ public class TestObjectStoreSchemaMethods { Assert.assertEquals("b", cols.get(1).getName()); Assert.assertEquals(ColumnType.FLOAT_TYPE_NAME, cols.get(1).getType()); - objectStore.dropSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version)); - schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version)); + objectStore.dropSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(db.getCatalogName(), db.getName(), schemaName), version)); + schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(db.getCatalogName(), db.getName(), schemaName), version)); Assert.assertNull(schemaVersion); } // Test that adding multiple versions of the same schema @Test public void multipleSchemaVersions() throws TException { - String dbName = createUniqueDatabaseForTest(); + Database db = createUniqueDatabaseForTest(); String schemaName = "schema195"; ISchema schema = new ISchemaBuilder() .setSchemaType(SchemaType.AVRO) .setName(schemaName) - .setDbName(dbName) + .inDb(db) .build(); objectStore.createISchema(schema); SchemaVersion schemaVersion = new SchemaVersionBuilder() @@ -290,7 +294,7 @@ public class TestObjectStoreSchemaMethods { .build(); objectStore.addSchemaVersion(schemaVersion); - schemaVersion = objectStore.getLatestSchemaVersion(new ISchemaName(dbName, schemaName)); + schemaVersion = objectStore.getLatestSchemaVersion(new ISchemaName(db.getCatalogName(), db.getName(), schemaName)); Assert.assertEquals(3, schemaVersion.getVersion()); Assert.assertEquals(3, schemaVersion.getColsSize()); List<FieldSchema> cols = schemaVersion.getCols(); @@ -302,14 +306,14 @@ public class TestObjectStoreSchemaMethods { Assert.assertEquals(ColumnType.DATE_TYPE_NAME, cols.get(1).getType()); Assert.assertEquals(ColumnType.TIMESTAMP_TYPE_NAME, cols.get(2).getType()); - schemaVersion = objectStore.getLatestSchemaVersion(new ISchemaName(dbName, "no.such.schema.with.this.name")); + schemaVersion = objectStore.getLatestSchemaVersion(new ISchemaName(db.getCatalogName(), db.getName(), "no.such.schema.with.this.name")); Assert.assertNull(schemaVersion); List<SchemaVersion> versions = - objectStore.getAllSchemaVersion(new ISchemaName(dbName, "there.really.isnt.a.schema.named.this")); + objectStore.getAllSchemaVersion(new ISchemaName(db.getCatalogName(), db.getName(), "there.really.isnt.a.schema.named.this")); Assert.assertNull(versions); - versions = objectStore.getAllSchemaVersion(new ISchemaName(dbName, schemaName)); + versions = objectStore.getAllSchemaVersion(new ISchemaName(db.getCatalogName(), db.getName(), schemaName)); Assert.assertEquals(3, versions.size()); versions.sort(Comparator.comparingInt(SchemaVersion::getVersion)); Assert.assertEquals(1, versions.get(0).getVersion()); @@ -339,16 +343,16 @@ public class TestObjectStoreSchemaMethods { @Test(expected = AlreadyExistsException.class) public void addDuplicateSchemaVersion() throws TException { - String dbName = createUniqueDatabaseForTest(); + Database db = createUniqueDatabaseForTest(); String schemaName = "schema1234"; int version = 1; - SchemaVersion schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version)); + SchemaVersion schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(db.getCatalogName(), db.getName(), schemaName), version)); Assert.assertNull(schemaVersion); ISchema schema = new ISchemaBuilder() .setSchemaType(SchemaType.AVRO) .setName(schemaName) - .setDbName(dbName) + .inDb(db) .build(); objectStore.createISchema(schema); @@ -365,16 +369,16 @@ public class TestObjectStoreSchemaMethods { @Test public void alterSchemaVersion() throws TException { - String dbName = createUniqueDatabaseForTest(); + Database db = createUniqueDatabaseForTest(); String schemaName = "schema371234"; int version = 1; - SchemaVersion schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version)); + SchemaVersion schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(db.getCatalogName(), db.getName(), schemaName), version)); Assert.assertNull(schemaVersion); ISchema schema = new ISchemaBuilder() .setSchemaType(SchemaType.AVRO) .setName(schemaName) - .setDbName(dbName) + .inDb(db) .build(); objectStore.createISchema(schema); @@ -387,10 +391,11 @@ public class TestObjectStoreSchemaMethods { .build(); objectStore.addSchemaVersion(schemaVersion); - schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version)); + schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(db.getCatalogName(), db.getName(), schemaName), version)); Assert.assertNotNull(schemaVersion); Assert.assertEquals(schemaName, schemaVersion.getSchema().getSchemaName()); - Assert.assertEquals(dbName, schemaVersion.getSchema().getDbName()); + Assert.assertEquals(db.getName(), schemaVersion.getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), schemaVersion.getSchema().getCatName()); Assert.assertEquals(version, schemaVersion.getVersion()); Assert.assertEquals(SchemaVersionState.INITIATED, schemaVersion.getState()); @@ -402,12 +407,13 @@ public class TestObjectStoreSchemaMethods { serde.setSerializerClass(serializer); serde.setDeserializerClass(deserializer); schemaVersion.setSerDe(serde); - objectStore.alterSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version), schemaVersion); + objectStore.alterSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(db.getCatalogName(), db.getName(), schemaName), version), schemaVersion); - schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(dbName, schemaName), version)); + schemaVersion = objectStore.getSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(db.getCatalogName(), db.getName(), schemaName), version)); Assert.assertNotNull(schemaVersion); Assert.assertEquals(schemaName, schemaVersion.getSchema().getSchemaName()); - Assert.assertEquals(dbName, schemaVersion.getSchema().getDbName()); + Assert.assertEquals(db.getName(), schemaVersion.getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), schemaVersion.getSchema().getCatName()); Assert.assertEquals(version, schemaVersion.getVersion()); Assert.assertEquals(SchemaVersionState.REVIEWED, schemaVersion.getState()); Assert.assertEquals(serdeName, schemaVersion.getSerDe().getName()); @@ -428,22 +434,22 @@ public class TestObjectStoreSchemaMethods { .addCol("b", ColumnType.FLOAT_TYPE_NAME) .setState(SchemaVersionState.INITIATED) .build(); - objectStore.alterSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(DEFAULT_DATABASE_NAME, schemaName), version), schemaVersion); + objectStore.alterSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, schemaName), version), schemaVersion); } @Test(expected = NoSuchObjectException.class) public void dropNonExistentSchemaVersion() throws NoSuchObjectException, MetaException { - objectStore.dropSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(DEFAULT_DATABASE_NAME, "ther is no schema named this"), 23)); + objectStore.dropSchemaVersion(new SchemaVersionDescriptor(new ISchemaName(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME, "ther is no schema named this"), 23)); } @Test public void schemaQuery() throws TException { - String dbName = createUniqueDatabaseForTest(); + Database db = createUniqueDatabaseForTest(); String schemaName1 = "a_schema1"; ISchema schema1 = new ISchemaBuilder() .setSchemaType(SchemaType.AVRO) .setName(schemaName1) - .setDbName(dbName) + .inDb(db) .build(); objectStore.createISchema(schema1); @@ -451,7 +457,7 @@ public class TestObjectStoreSchemaMethods { ISchema schema2 = new ISchemaBuilder() .setSchemaType(SchemaType.AVRO) .setName(schemaName2) - .setDbName(dbName) + .inDb(db) .build(); objectStore.createISchema(schema2); @@ -497,7 +503,8 @@ public class TestObjectStoreSchemaMethods { results = objectStore.getSchemaVersionsByColumns("gamma", null, null); Assert.assertEquals(1, results.size()); Assert.assertEquals(schemaName1, results.get(0).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(0).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(0).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(0).getSchema().getCatName()); Assert.assertEquals(2, results.get(0).getVersion()); // fetch 2 in same schema @@ -505,10 +512,12 @@ public class TestObjectStoreSchemaMethods { Assert.assertEquals(2, results.size()); Collections.sort(results); Assert.assertEquals(schemaName1, results.get(0).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(0).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(0).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(0).getSchema().getCatName()); Assert.assertEquals(1, results.get(0).getVersion()); Assert.assertEquals(schemaName1, results.get(1).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(1).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(1).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(1).getSchema().getCatName()); Assert.assertEquals(2, results.get(1).getVersion()); // fetch across schemas @@ -516,16 +525,20 @@ public class TestObjectStoreSchemaMethods { Assert.assertEquals(4, results.size()); Collections.sort(results); Assert.assertEquals(schemaName1, results.get(0).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(0).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(0).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(0).getSchema().getCatName()); Assert.assertEquals(1, results.get(0).getVersion()); Assert.assertEquals(schemaName1, results.get(1).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(1).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(1).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(1).getSchema().getCatName()); Assert.assertEquals(2, results.get(1).getVersion()); Assert.assertEquals(schemaName2, results.get(2).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(2).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(2).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(2).getSchema().getCatName()); Assert.assertEquals(1, results.get(2).getVersion()); Assert.assertEquals(schemaName2, results.get(3).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(3).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(3).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(3).getSchema().getCatName()); Assert.assertEquals(2, results.get(3).getVersion()); // fetch by namespace @@ -533,10 +546,12 @@ public class TestObjectStoreSchemaMethods { Assert.assertEquals(2, results.size()); Collections.sort(results); Assert.assertEquals(schemaName1, results.get(0).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(0).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(0).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(0).getSchema().getCatName()); Assert.assertEquals(2, results.get(0).getVersion()); Assert.assertEquals(schemaName2, results.get(1).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(1).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(1).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(1).getSchema().getCatName()); Assert.assertEquals(2, results.get(1).getVersion()); // fetch by name and type @@ -544,10 +559,12 @@ public class TestObjectStoreSchemaMethods { Assert.assertEquals(2, results.size()); Collections.sort(results); Assert.assertEquals(schemaName2, results.get(0).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(0).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(0).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(0).getSchema().getCatName()); Assert.assertEquals(1, results.get(0).getVersion()); Assert.assertEquals(schemaName2, results.get(1).getSchema().getSchemaName()); - Assert.assertEquals(dbName, results.get(1).getSchema().getDbName()); + Assert.assertEquals(db.getName(), results.get(1).getSchema().getDbName()); + Assert.assertEquals(db.getCatalogName(), results.get(1).getSchema().getCatName()); Assert.assertEquals(2, results.get(1).getVersion()); // Make sure matching name but wrong type doesn't return @@ -560,14 +577,26 @@ public class TestObjectStoreSchemaMethods { } private static int dbNum = 1; - private String createUniqueDatabaseForTest() throws MetaException, InvalidObjectException { + private static Random rand = new Random(); + private Database createUniqueDatabaseForTest() throws MetaException, InvalidObjectException { + String catName; + if (rand.nextDouble() < 0.5) { + catName = "unique_cat_for_test_" + dbNum++; + objectStore.createCatalog(new CatalogBuilder() + .setName(catName) + .setLocation("there") + .build()); + } else { + catName = DEFAULT_CATALOG_NAME; + } String dbName = "uniquedbfortest" + dbNum++; Database db = new DatabaseBuilder() .setName(dbName) + .setCatalogName(catName) .setLocation("somewhere") .setDescription("descriptive") - .build(); + .build(conf); objectStore.createDatabase(db); - return dbName; + return db; } }
http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestOldSchema.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestOldSchema.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestOldSchema.java index f286da8..49033d3 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestOldSchema.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestOldSchema.java @@ -26,6 +26,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.ndv.hll.HyperLogLog; import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.apache.hadoop.hive.metastore.api.AggrStats; +import org.apache.hadoop.hive.metastore.api.Catalog; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; @@ -42,6 +43,7 @@ import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.junit.After; @@ -52,9 +54,12 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; + @Category(MetastoreUnitTest.class) public class TestOldSchema { private ObjectStore store = null; + private Configuration conf; private static final Logger LOG = LoggerFactory.getLogger(TestOldSchema.class.getName()); @@ -91,13 +96,14 @@ public class TestOldSchema { @Before public void setUp() throws Exception { - Configuration conf = MetastoreConf.newMetastoreConf(); + conf = MetastoreConf.newMetastoreConf(); MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_BITVECTOR, false); MetaStoreTestUtils.setConfForStandloneMode(conf); store = new ObjectStore(); store.setConf(conf); dropAllStoreObjects(store); + HiveMetaStore.HMSHandler.createDefaultCatalog(store, new Warehouse(conf)); HyperLogLog hll = HyperLogLog.builder().build(); hll.addLong(1); @@ -121,7 +127,11 @@ public class TestOldSchema { public void testPartitionOps() throws Exception { String dbName = "default"; String tableName = "snp"; - Database db1 = new Database(dbName, "description", "locationurl", null); + Database db1 = new DatabaseBuilder() + .setName(dbName) + .setDescription("description") + .setLocation("locationurl") + .build(conf); store.createDatabase(db1); long now = System.currentTimeMillis(); List<FieldSchema> cols = new ArrayList<>(); @@ -143,6 +153,7 @@ public class TestOldSchema { psd.setLocation("file:/tmp/default/hit/ds=" + partVal); Partition part = new Partition(partVal, dbName, tableName, (int) now, (int) now, psd, Collections.emptyMap()); + part.setCatName(DEFAULT_CATALOG_NAME); store.addPartition(part); ColumnStatistics cs = new ColumnStatistics(); ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, dbName, tableName); @@ -185,7 +196,7 @@ public class TestOldSchema { for (int i = 0; i < 10; i++) { partNames.add("ds=" + i); } - AggrStats aggrStats = store.get_aggr_stats_for(dbName, tableName, partNames, + AggrStats aggrStats = store.get_aggr_stats_for(DEFAULT_CATALOG_NAME, dbName, tableName, partNames, Arrays.asList("col1")); statChecker.checkStats(aggrStats); @@ -200,18 +211,18 @@ public class TestOldSchema { try { Deadline.registerIfNot(100000); Deadline.startTimer("getPartition"); - List<String> dbs = store.getAllDatabases(); + List<String> dbs = store.getAllDatabases(DEFAULT_CATALOG_NAME); for (int i = 0; i < dbs.size(); i++) { String db = dbs.get(i); - List<String> tbls = store.getAllTables(db); + List<String> tbls = store.getAllTables(DEFAULT_CATALOG_NAME, db); for (String tbl : tbls) { - List<Partition> parts = store.getPartitions(db, tbl, 100); + List<Partition> parts = store.getPartitions(DEFAULT_CATALOG_NAME, db, tbl, 100); for (Partition part : parts) { - store.dropPartition(db, tbl, part.getValues()); + store.dropPartition(DEFAULT_CATALOG_NAME, db, tbl, part.getValues()); } - store.dropTable(db, tbl); + store.dropTable(DEFAULT_CATALOG_NAME, db, tbl); } - store.dropDatabase(db); + store.dropDatabase(DEFAULT_CATALOG_NAME, db); } } catch (NoSuchObjectException e) { } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java index 6b7d913..3d1723e 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; @@ -57,9 +58,9 @@ public class TestRemoteHiveMetaStoreIpAddress { @Test public void testIpAddress() throws Exception { - Database db = new Database(); - db.setName("testIpAddressIp"); - msc.createDatabase(db); + Database db = new DatabaseBuilder() + .setName("testIpAddressIp") + .create(msc, conf); msc.dropDatabase(db.getName()); } } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java index 11f84f2..930e996 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; @@ -63,19 +64,17 @@ public class TestRetryingHMSHandler { String dbName = "hive4159"; String tblName = "tmptbl"; - Database db = new Database(); - db.setName(dbName); - msc.createDatabase(db); + new DatabaseBuilder() + .setName(dbName) + .create(msc, conf); Assert.assertEquals(2, AlternateFailurePreListener.getCallCount()); - Table tbl = new TableBuilder() + new TableBuilder() .setDbName(dbName) .setTableName(tblName) .addCol("c1", ColumnType.STRING_TYPE_NAME) - .build(); - - msc.createTable(tbl); + .create(msc, conf); Assert.assertEquals(4, AlternateFailurePreListener.getCallCount()); } http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestStats.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestStats.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestStats.java new file mode 100644 index 0000000..6cca062 --- /dev/null +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestStats.java @@ -0,0 +1,728 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest; +import org.apache.hadoop.hive.metastore.api.AggrStats; +import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; +import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; +import org.apache.hadoop.hive.metastore.api.Catalog; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.Date; +import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; +import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; +import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; +import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; +import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.client.builder.CatalogBuilder; +import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; +import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; +import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; + +@Category(MetastoreCheckinTest.class) +public class TestStats { + private static final Logger LOG = LoggerFactory.getLogger(TestStats.class); + + private static final String NO_CAT = "DO_NOT_USE_A_CATALOG!"; + + private IMetaStoreClient client; + private Configuration conf; + + @Before + public void setUp() throws MetaException { + conf = MetastoreConf.newMetastoreConf(); + MetaStoreTestUtils.setConfForStandloneMode(conf); + MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.AGGREGATE_STATS_CACHE_ENABLED, false); + // Get new client + client = new HiveMetaStoreClient(conf); + } + + @After + public void tearDown() throws TException { + // Drop any left over catalogs + List<String> catalogs = client.getCatalogs(); + for (String catName : catalogs) { + if (!catName.equalsIgnoreCase(DEFAULT_CATALOG_NAME)) { + // First drop any databases in catalog + List<String> databases = client.getAllDatabases(catName); + for (String db : databases) { + client.dropDatabase(catName, db, true, false, true); + } + client.dropCatalog(catName); + } else { + List<String> databases = client.getAllDatabases(catName); + for (String db : databases) { + if (!db.equalsIgnoreCase(Warehouse.DEFAULT_DATABASE_NAME)) { + client.dropDatabase(catName, db, true, false, true); + } + } + } + } + try { + if (client != null) { + client.close(); + } + } finally { + client = null; + } + } + + private Map<String, Column> buildAllColumns() { + Map<String, Column> colMap = new HashMap<>(6); + Column[] cols = { new BinaryColumn(), new BooleanColumn(), new DateColumn(), + new DoubleColumn(), new LongColumn(), new StringColumn() }; + for (Column c : cols) colMap.put(c.colName, c); + return colMap; + } + + private List<String> createMetadata(String catName, String dbName, String tableName, + String partKey, List<String> partVals, + Map<String, Column> colMap) + throws TException { + if (!DEFAULT_CATALOG_NAME.equals(catName) && !NO_CAT.equals(catName)) { + Catalog cat = new CatalogBuilder() + .setName(catName) + .setLocation(MetaStoreTestUtils.getTestWarehouseDir(catName)) + .build(); + client.createCatalog(cat); + } + + Database db; + if (!DEFAULT_DATABASE_NAME.equals(dbName)) { + DatabaseBuilder dbBuilder = new DatabaseBuilder() + .setName(dbName); + if (!NO_CAT.equals(catName)) dbBuilder.setCatalogName(catName); + db = dbBuilder.create(client, conf); + } else { + db = client.getDatabase(DEFAULT_CATALOG_NAME, DEFAULT_DATABASE_NAME); + } + + TableBuilder tb = new TableBuilder() + .inDb(db) + .setTableName(tableName); + + for (Column col : colMap.values()) tb.addCol(col.colName, col.colType); + + if (partKey != null) { + assert partVals != null && !partVals.isEmpty() : + "Must provide partition values for partitioned table"; + tb.addPartCol(partKey, ColumnType.STRING_TYPE_NAME); + } + Table table = tb.create(client, conf); + + if (partKey != null) { + for (String partVal : partVals) { + new PartitionBuilder() + .inTable(table) + .addValue(partVal) + .addToTable(client, conf); + } + } + + SetPartitionsStatsRequest rqst = new SetPartitionsStatsRequest(); + List<String> partNames = new ArrayList<>(); + if (partKey == null) { + rqst.addToColStats(buildStatsForOneTableOrPartition(catName, dbName, tableName, null, + colMap.values())); + } else { + for (String partVal : partVals) { + String partName = partKey + "=" + partVal; + rqst.addToColStats(buildStatsForOneTableOrPartition(catName, dbName, tableName, partName, + colMap.values())); + partNames.add(partName); + } + } + client.setPartitionColumnStatistics(rqst); + return partNames; + } + + private ColumnStatistics buildStatsForOneTableOrPartition(String catName, String dbName, + String tableName, String partName, + Collection<Column> cols) { + ColumnStatisticsDesc desc = new ColumnStatisticsDesc(partName == null, dbName, tableName); + if (!NO_CAT.equals(catName)) desc.setCatName(catName); + if (partName != null) desc.setPartName(partName); + + List<ColumnStatisticsObj> objs = new ArrayList<>(cols.size()); + + for (Column col : cols) objs.add(col.generate()); + + return new ColumnStatistics(desc, objs); + } + + private void dropStats(String catName, String dbName, String tableName, String partName, + Collection<String> colNames) + throws TException { + for (String colName : colNames) { + if (partName == null) { + if (NO_CAT.equals(catName)) client.deleteTableColumnStatistics(dbName, tableName, colName); + else client.deleteTableColumnStatistics(catName, dbName, tableName, colName); + } else { + if (NO_CAT.equals(catName)) client.deletePartitionColumnStatistics(dbName, tableName, partName, colName); + else client.deletePartitionColumnStatistics(catName, dbName, tableName, partName, colName); + } + } + } + + private void compareStatsForTable(String catName, String dbName, String tableName, + Map<String, Column> colMap) throws TException { + List<ColumnStatisticsObj> objs = catName.equals(NO_CAT) ? + client.getTableColumnStatistics(dbName, tableName, new ArrayList<>(colMap.keySet())) : + client.getTableColumnStatistics(catName, dbName, tableName, new ArrayList<>(colMap.keySet())); + compareStatsForOneTableOrPartition(objs, 0, colMap); + } + + private void compareStatsForPartitions(String catName, String dbName, String tableName, + List<String> partNames, final Map<String, Column> colMap) + throws TException { + Map<String, List<ColumnStatisticsObj>> partObjs = catName.equals(NO_CAT) ? + client.getPartitionColumnStatistics(dbName, tableName, partNames, new ArrayList<>(colMap.keySet())) : + client.getPartitionColumnStatistics(catName, dbName, tableName, partNames, new ArrayList<>(colMap.keySet())); + for (int i = 0; i < partNames.size(); i++) { + compareStatsForOneTableOrPartition(partObjs.get(partNames.get(i)), i, colMap); + } + AggrStats aggr = catName.equals(NO_CAT) ? + client.getAggrColStatsFor(dbName, tableName, new ArrayList<>(colMap.keySet()), partNames) : + client.getAggrColStatsFor(catName, dbName, tableName, new ArrayList<>(colMap.keySet()), partNames); + Assert.assertEquals(partNames.size(), aggr.getPartsFound()); + Assert.assertEquals(colMap.size(), aggr.getColStatsSize()); + aggr.getColStats().forEach(cso -> colMap.get(cso.getColName()).compareAggr(cso)); + } + + private void compareStatsForOneTableOrPartition(List<ColumnStatisticsObj> objs, + final int partOffset, + final Map<String, Column> colMap) + throws TException { + Assert.assertEquals(objs.size(), colMap.size()); + objs.forEach(cso -> colMap.get(cso.getColName()).compare(cso, partOffset)); + } + + @Test + public void tableInHiveCatalog() throws TException { + String dbName = "db_table_stats"; + String tableName = "table_in_default_db_stats"; + Map<String, Column> colMap = buildAllColumns(); + createMetadata(DEFAULT_CATALOG_NAME, dbName, tableName, null, null, colMap); + compareStatsForTable(DEFAULT_CATALOG_NAME, dbName, tableName, colMap); + dropStats(DEFAULT_CATALOG_NAME, dbName, tableName, null, colMap.keySet()); + } + + @Test + public void partitionedTableInHiveCatalog() throws TException { + String dbName = "db_part_stats"; + String tableName = "partitioned_table_in_default_db_stats"; + Map<String, Column> colMap = buildAllColumns(); + List<String> partNames = createMetadata(DEFAULT_CATALOG_NAME, dbName, tableName, "pk", + Arrays.asList("a1", "a2", "a3"), colMap); + compareStatsForPartitions(DEFAULT_CATALOG_NAME, dbName, tableName, partNames, colMap); + for (String partName : partNames) { + dropStats(DEFAULT_CATALOG_NAME, dbName, tableName, partName, colMap.keySet()); + } + } + + @Test + public void tableOtherCatalog() throws TException { + String catName = "cat_table_stats"; + String dbName = "other_cat_db_table_stats"; + String tableName = "table_in_default_db_stats"; + Map<String, Column> colMap = buildAllColumns(); + createMetadata(catName, dbName, tableName, null, null, colMap); + compareStatsForTable(catName, dbName, tableName, colMap); + dropStats(catName, dbName, tableName, null, colMap.keySet()); + } + + @Test + public void partitionedTableOtherCatalog() throws TException { + String catName = "cat_table_stats"; + String dbName = "other_cat_db_part_stats"; + String tableName = "partitioned_table_in_default_db_stats"; + Map<String, Column> colMap = buildAllColumns(); + List<String> partNames = createMetadata(catName, dbName, tableName, "pk", + Arrays.asList("a1", "a2", "a3"), colMap); + compareStatsForPartitions(catName, dbName, tableName, partNames, colMap); + for (String partName : partNames) { + dropStats(catName, dbName, tableName, partName, colMap.keySet()); + } + } + + @Test + public void tableDeprecatedCalls() throws TException { + String dbName = "old_db_table_stats"; + String tableName = "table_in_default_db_stats"; + Map<String, Column> colMap = buildAllColumns(); + createMetadata(NO_CAT, dbName, tableName, null, null, colMap); + compareStatsForTable(NO_CAT, dbName, tableName, colMap); + dropStats(NO_CAT, dbName, tableName, null, colMap.keySet()); + } + + @Test + public void partitionedTableDeprecatedCalls() throws TException { + String dbName = "old_db_part_stats"; + String tableName = "partitioned_table_in_default_db_stats"; + Map<String, Column> colMap = buildAllColumns(); + List<String> partNames = createMetadata(NO_CAT, dbName, tableName, "pk", + Arrays.asList("a1", "a2", "a3"), colMap); + compareStatsForPartitions(NO_CAT, dbName, tableName, partNames, colMap); + for (String partName : partNames) { + dropStats(NO_CAT, dbName, tableName, partName, colMap.keySet()); + } + } + + private abstract class Column { + final String colName; + final String colType; + + Random rand = new Random(); + + List<Long> maxLens, numNulls, numDvs; + List<Double> avgLens; + + + public Column(String colName, String colType) { + this.colName = colName; + this.colType = colType; + maxLens = new ArrayList<>(); + numNulls = new ArrayList<>(); + avgLens = new ArrayList<>(); + numDvs = new ArrayList<>(); + } + + abstract ColumnStatisticsObj generate(); + abstract void compare(ColumnStatisticsObj obj, int offset); + abstract void compareAggr(ColumnStatisticsObj obj); + + void compareCommon(ColumnStatisticsObj obj) { + Assert.assertEquals(colName, obj.getColName()); + Assert.assertEquals(colType, obj.getColType()); + } + + long genMaxLen() { + return genPositiveLong(maxLens); + } + + long getMaxLen() { + return maxLong(maxLens); + } + + long genNumNulls() { + return genPositiveLong(numNulls); + } + + long genNumDvs() { + return genPositiveLong(numDvs); + } + + long getNumNulls() { + return sumLong(numNulls); + } + + long getNumDvs() { + return maxLong(numDvs); + } + + double genAvgLens() { + return genDouble(avgLens); + } + + double getAvgLen() { + return maxDouble(avgLens); + } + + protected long genNegativeLong(List<Long> addTo) { + long val = rand.nextInt(100); + if (val > 0) val *= -1; + addTo.add(val); + return val; + } + + protected long genPositiveLong(List<Long> addTo) { + long val = rand.nextInt(100); + val = Math.abs(val) + 1; // make sure it isn't 0 + addTo.add(val); + return val; + } + + protected long maxLong(List<Long> maxOf) { + long max = Long.MIN_VALUE; + for (long maybe : maxOf) max = Math.max(max, maybe); + return max; + } + + protected long sumLong(List<Long> sumOf) { + long sum = 0; + for (long element : sumOf) sum += element; + return sum; + } + + protected double genDouble(List<Double> addTo) { + double val = rand.nextDouble() * rand.nextInt(100); + addTo.add(val); + return val; + } + + protected double maxDouble(List<Double> maxOf) { + double max = Double.MIN_VALUE; + for (double maybe : maxOf) max = Math.max(max, maybe); + return max; + } + + } + + private class BinaryColumn extends Column { + public BinaryColumn() { + super("bincol", ColumnType.BINARY_TYPE_NAME); + } + + @Override + ColumnStatisticsObj generate() { + BinaryColumnStatsData binData = new BinaryColumnStatsData(genMaxLen(), genAvgLens(), genNumNulls()); + ColumnStatisticsData data = new ColumnStatisticsData(); + data.setBinaryStats(binData); + return new ColumnStatisticsObj(colName, colType, data); + } + + @Override + void compare(ColumnStatisticsObj obj, int offset) { + compareCommon(obj); + Assert.assertEquals("binary max length", maxLens.get(offset), + (Long)obj.getStatsData().getBinaryStats().getMaxColLen()); + Assert.assertEquals("binary min length", avgLens.get(offset), obj.getStatsData().getBinaryStats().getAvgColLen(), 0.01); + Assert.assertEquals("binary num nulls", numNulls.get(offset), (Long)obj.getStatsData().getBinaryStats().getNumNulls()); + } + + @Override + void compareAggr(ColumnStatisticsObj obj) { + compareCommon(obj); + Assert.assertEquals("aggr binary max length", getMaxLen(), obj.getStatsData().getBinaryStats().getMaxColLen()); + Assert.assertEquals("aggr binary min length", getAvgLen(), obj.getStatsData().getBinaryStats().getAvgColLen(), 0.01); + Assert.assertEquals("aggr binary num nulls", getNumNulls(), obj.getStatsData().getBinaryStats().getNumNulls()); + } + } + + private class BooleanColumn extends Column { + private List<Long> numTrues, numFalses; + + public BooleanColumn() { + super("boolcol", ColumnType.BOOLEAN_TYPE_NAME); + numTrues = new ArrayList<>(); + numFalses = new ArrayList<>(); + } + + @Override + ColumnStatisticsObj generate() { + BooleanColumnStatsData + boolData = new BooleanColumnStatsData(genNumTrues(), genNumFalses(), genNumNulls()); + ColumnStatisticsData data = new ColumnStatisticsData(); + data.setBooleanStats(boolData); + return new ColumnStatisticsObj(colName, colType, data); + } + + @Override + void compare(ColumnStatisticsObj obj, int offset) { + compareCommon(obj); + Assert.assertEquals("boolean num trues", numTrues.get(offset), (Long)obj.getStatsData().getBooleanStats().getNumTrues()); + Assert.assertEquals("boolean num falses", numFalses.get(offset), (Long)obj.getStatsData().getBooleanStats().getNumFalses()); + Assert.assertEquals("boolean num nulls", numNulls.get(offset), (Long)obj.getStatsData().getBooleanStats().getNumNulls()); + } + + @Override + void compareAggr(ColumnStatisticsObj obj) { + compareCommon(obj); + Assert.assertEquals("aggr boolean num trues", getNumTrues(), obj.getStatsData().getBooleanStats().getNumTrues()); + Assert.assertEquals("aggr boolean num falses", getNumFalses(), obj.getStatsData().getBooleanStats().getNumFalses()); + Assert.assertEquals("aggr boolean num nulls", getNumNulls(), obj.getStatsData().getBooleanStats().getNumNulls()); + } + + private long genNumTrues() { + return genPositiveLong(numTrues); + } + + private long genNumFalses() { + return genPositiveLong(numFalses); + } + + private long getNumTrues() { + return sumLong(numTrues); + } + + private long getNumFalses() { + return sumLong(numFalses); + } + } + + private class DateColumn extends Column { + private List<Date> lowVals, highVals; + + public DateColumn() { + super("datecol", ColumnType.DATE_TYPE_NAME); + lowVals = new ArrayList<>(); + highVals = new ArrayList<>(); + } + + @Override + ColumnStatisticsObj generate() { + DateColumnStatsData dateData = new DateColumnStatsData(genNumNulls(), genNumDvs()); + dateData.setLowValue(genLowValue()); + dateData.setHighValue(genHighValue()); + ColumnStatisticsData data = new ColumnStatisticsData(); + data.setDateStats(dateData); + return new ColumnStatisticsObj(colName, colType, data); + } + + @Override + void compare(ColumnStatisticsObj obj, int offset) { + compareCommon(obj); + Assert.assertEquals("date num nulls", numNulls.get(offset), (Long)obj.getStatsData().getDateStats().getNumNulls()); + Assert.assertEquals("date num dvs", numDvs.get(offset), (Long)obj.getStatsData().getDateStats().getNumDVs()); + Assert.assertEquals("date low val", lowVals.get(offset), obj.getStatsData().getDateStats().getLowValue()); + Assert.assertEquals("date high val", highVals.get(offset), obj.getStatsData().getDateStats().getHighValue()); + } + + @Override + void compareAggr(ColumnStatisticsObj obj) { + compareCommon(obj); + Assert.assertEquals("aggr date num nulls", getNumNulls(), obj.getStatsData().getDateStats().getNumNulls()); + Assert.assertEquals("aggr date num dvs", getNumDvs(), obj.getStatsData().getDateStats().getNumDVs()); + Assert.assertEquals("aggr date low val", getLowVal(), obj.getStatsData().getDateStats().getLowValue()); + Assert.assertEquals("aggr date high val", getHighVal(), obj.getStatsData().getDateStats().getHighValue()); + } + + private Date genLowValue() { + Date d = new Date(rand.nextInt(100) * -1); + lowVals.add(d); + return d; + } + + private Date genHighValue() { + Date d = new Date(rand.nextInt(200)); + highVals.add(d); + return d; + } + + private Date getLowVal() { + long min = Long.MAX_VALUE; + for (Date d : lowVals) min = Math.min(min, d.getDaysSinceEpoch()); + return new Date(min); + } + + private Date getHighVal() { + long max = Long.MIN_VALUE; + for (Date d : highVals) max = Math.max(max, d.getDaysSinceEpoch()); + return new Date(max); + } + } + + private class DoubleColumn extends Column { + List<Double> lowVals, highVals; + + public DoubleColumn() { + super("doublecol", ColumnType.DOUBLE_TYPE_NAME); + lowVals = new ArrayList<>(); + highVals = new ArrayList<>(); + } + + @Override + ColumnStatisticsObj generate() { + DoubleColumnStatsData doubleData = new DoubleColumnStatsData(genNumNulls(), genNumDvs()); + doubleData.setLowValue(genLowVal()); + doubleData.setHighValue(genHighVal()); + ColumnStatisticsData data = new ColumnStatisticsData(); + data.setDoubleStats(doubleData); + return new ColumnStatisticsObj(colName, colType, data); + } + + @Override + void compare(ColumnStatisticsObj obj, int offset) { + compareCommon(obj); + Assert.assertEquals("double num nulls", numNulls.get(offset), + (Long)obj.getStatsData().getDoubleStats().getNumNulls()); + Assert.assertEquals("double num dvs", numDvs.get(offset), + (Long)obj.getStatsData().getDoubleStats().getNumDVs()); + Assert.assertEquals("double low val", lowVals.get(offset), + obj.getStatsData().getDoubleStats().getLowValue(), 0.01); + Assert.assertEquals("double high val", highVals.get(offset), + obj.getStatsData().getDoubleStats().getHighValue(), 0.01); + } + + @Override + void compareAggr(ColumnStatisticsObj obj) { + compareCommon(obj); + Assert.assertEquals("aggr double num nulls", getNumNulls(), + obj.getStatsData().getDoubleStats().getNumNulls()); + Assert.assertEquals("aggr double num dvs", getNumDvs(), + obj.getStatsData().getDoubleStats().getNumDVs()); + Assert.assertEquals("aggr double low val", getLowVal(), + obj.getStatsData().getDoubleStats().getLowValue(), 0.01); + Assert.assertEquals("aggr double high val", getHighVal(), + obj.getStatsData().getDoubleStats().getHighValue(), 0.01); + + } + + private double genLowVal() { + return genDouble(lowVals); + } + + private double genHighVal() { + return genDouble(highVals); + } + + private double getLowVal() { + double min = Double.MAX_VALUE; + for (Double d : lowVals) min = Math.min(min, d); + return min; + } + + private double getHighVal() { + return maxDouble(highVals); + } + } + + private class LongColumn extends Column { + List<Long> lowVals, highVals; + + public LongColumn() { + super("bigintcol", ColumnType.BIGINT_TYPE_NAME); + lowVals = new ArrayList<>(); + highVals = new ArrayList<>(); + } + + @Override + ColumnStatisticsObj generate() { + LongColumnStatsData longData = new LongColumnStatsData(genNumNulls(), genNumDvs()); + longData.setLowValue(genLowVal()); + longData.setHighValue(genHighVal()); + ColumnStatisticsData data = new ColumnStatisticsData(); + data.setLongStats(longData); + return new ColumnStatisticsObj(colName, colType, data); + } + + @Override + void compare(ColumnStatisticsObj obj, int offset) { + compareCommon(obj); + Assert.assertEquals("long num nulls", numNulls.get(offset), + (Long)obj.getStatsData().getLongStats().getNumNulls()); + Assert.assertEquals("long num dvs", numDvs.get(offset), + (Long)obj.getStatsData().getLongStats().getNumDVs()); + Assert.assertEquals("long low val", (long)lowVals.get(offset), + obj.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals("long high val", (long)highVals.get(offset), + obj.getStatsData().getLongStats().getHighValue()); + } + + @Override + void compareAggr(ColumnStatisticsObj obj) { + compareCommon(obj); + Assert.assertEquals("aggr long num nulls", getNumNulls(), + obj.getStatsData().getLongStats().getNumNulls()); + Assert.assertEquals("aggr long num dvs", getNumDvs(), + obj.getStatsData().getLongStats().getNumDVs()); + Assert.assertEquals("aggr long low val", getLowVal(), + obj.getStatsData().getLongStats().getLowValue()); + Assert.assertEquals("aggr long high val", getHighVal(), + obj.getStatsData().getLongStats().getHighValue()); + } + + private long genLowVal() { + return genNegativeLong(lowVals); + } + + private long genHighVal() { + return genPositiveLong(highVals); + } + + private long getLowVal() { + long min = Long.MAX_VALUE; + for (Long val : lowVals) min = Math.min(min, val); + return min; + } + + private long getHighVal() { + return maxLong(highVals); + } + } + + private class StringColumn extends Column { + public StringColumn() { + super("strcol", ColumnType.STRING_TYPE_NAME); + } + + @Override + ColumnStatisticsObj generate() { + StringColumnStatsData strData = new StringColumnStatsData(genMaxLen(), genAvgLens(), + genNumNulls(), genNumDvs()); + ColumnStatisticsData data = new ColumnStatisticsData(); + data.setStringStats(strData); + return new ColumnStatisticsObj(colName, colType, data); + } + + @Override + void compare(ColumnStatisticsObj obj, int offset) { + compareCommon(obj); + Assert.assertEquals("str num nulls", numNulls.get(offset), + (Long)obj.getStatsData().getStringStats().getNumNulls()); + Assert.assertEquals("str num dvs", numDvs.get(offset), + (Long)obj.getStatsData().getStringStats().getNumDVs()); + Assert.assertEquals("str low val", (long)maxLens.get(offset), + obj.getStatsData().getStringStats().getMaxColLen()); + Assert.assertEquals("str high val", avgLens.get(offset), + obj.getStatsData().getStringStats().getAvgColLen(), 0.01); + } + + @Override + void compareAggr(ColumnStatisticsObj obj) { + compareCommon(obj); + Assert.assertEquals("aggr str num nulls", getNumNulls(), + obj.getStatsData().getStringStats().getNumNulls()); + Assert.assertEquals("aggr str num dvs", getNumDvs(), + obj.getStatsData().getStringStats().getNumDVs()); + Assert.assertEquals("aggr str low val", getMaxLen(), + obj.getStatsData().getStringStats().getMaxColLen()); + Assert.assertEquals("aggr str high val", getAvgLen(), + obj.getStatsData().getStringStats().getAvgColLen(), 0.01); + + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ba8a99e1/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java index 150b6ca..c9a6a47 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/VerifyingObjectStore.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.metastore; import static org.apache.commons.lang.StringUtils.repeat; +import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; import java.lang.reflect.AccessibleObject; import java.lang.reflect.Array; @@ -50,35 +51,36 @@ public class VerifyingObjectStore extends ObjectStore { } @Override - public List<Partition> getPartitionsByFilter(String dbName, String tblName, String filter, - short maxParts) throws MetaException, NoSuchObjectException { + public List<Partition> getPartitionsByFilter(String catName, String dbName, String tblName, + String filter, short maxParts) + throws MetaException, NoSuchObjectException { List<Partition> sqlResults = getPartitionsByFilterInternal( - dbName, tblName, filter, maxParts, true, false); + catName, dbName, tblName, filter, maxParts, true, false); List<Partition> ormResults = getPartitionsByFilterInternal( - dbName, tblName, filter, maxParts, false, true); + catName, dbName, tblName, filter, maxParts, false, true); verifyLists(sqlResults, ormResults, Partition.class); return sqlResults; } @Override - public List<Partition> getPartitionsByNames(String dbName, String tblName, + public List<Partition> getPartitionsByNames(String catName, String dbName, String tblName, List<String> partNames) throws MetaException, NoSuchObjectException { List<Partition> sqlResults = getPartitionsByNamesInternal( - dbName, tblName, partNames, true, false); + catName, dbName, tblName, partNames, true, false); List<Partition> ormResults = getPartitionsByNamesInternal( - dbName, tblName, partNames, false, true); + catName, dbName, tblName, partNames, false, true); verifyLists(sqlResults, ormResults, Partition.class); return sqlResults; } @Override - public boolean getPartitionsByExpr(String dbName, String tblName, byte[] expr, + public boolean getPartitionsByExpr(String catName, String dbName, String tblName, byte[] expr, String defaultPartitionName, short maxParts, List<Partition> result) throws TException { List<Partition> ormParts = new LinkedList<>(); boolean sqlResult = getPartitionsByExprInternal( - dbName, tblName, expr, defaultPartitionName, maxParts, result, true, false); + catName, dbName, tblName, expr, defaultPartitionName, maxParts, result, true, false); boolean ormResult = getPartitionsByExprInternal( - dbName, tblName, expr, defaultPartitionName, maxParts, ormParts, false, true); + catName, dbName, tblName, expr, defaultPartitionName, maxParts, ormParts, false, true); if (sqlResult != ormResult) { String msg = "The unknown flag is different - SQL " + sqlResult + ", ORM " + ormResult; LOG.error(msg); @@ -90,32 +92,32 @@ public class VerifyingObjectStore extends ObjectStore { @Override public List<Partition> getPartitions( - String dbName, String tableName, int maxParts) throws MetaException, NoSuchObjectException { - List<Partition> sqlResults = getPartitionsInternal(dbName, tableName, maxParts, true, false); - List<Partition> ormResults = getPartitionsInternal(dbName, tableName, maxParts, false, true); + String catName, String dbName, String tableName, int maxParts) throws MetaException, NoSuchObjectException { + List<Partition> sqlResults = getPartitionsInternal(catName, dbName, tableName, maxParts, true, false); + List<Partition> ormResults = getPartitionsInternal(catName, dbName, tableName, maxParts, false, true); verifyLists(sqlResults, ormResults, Partition.class); return sqlResults; } @Override - public ColumnStatistics getTableColumnStatistics(String dbName, + public ColumnStatistics getTableColumnStatistics(String catName, String dbName, String tableName, List<String> colNames) throws MetaException, NoSuchObjectException { ColumnStatistics sqlResult = getTableColumnStatisticsInternal( - dbName, tableName, colNames, true, false); + catName, dbName, tableName, colNames, true, false); ColumnStatistics jdoResult = getTableColumnStatisticsInternal( - dbName, tableName, colNames, false, true); + catName, dbName, tableName, colNames, false, true); verifyObjects(sqlResult, jdoResult, ColumnStatistics.class); return sqlResult; } @Override - public List<ColumnStatistics> getPartitionColumnStatistics(String dbName, + public List<ColumnStatistics> getPartitionColumnStatistics(String catName, String dbName, String tableName, List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException { List<ColumnStatistics> sqlResult = getPartitionColumnStatisticsInternal( - dbName, tableName, partNames, colNames, true, false); + catName, dbName, tableName, partNames, colNames, true, false); List<ColumnStatistics> jdoResult = getPartitionColumnStatisticsInternal( - dbName, tableName, partNames, colNames, false, true); + catName, dbName, tableName, partNames, colNames, false, true); verifyLists(sqlResult, jdoResult, ColumnStatistics.class); return sqlResult; }
