cshuo commented on code in PR #19393:
URL: https://github.com/apache/hudi/pull/19393#discussion_r3670565908
##########
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java:
##########
@@ -149,6 +164,259 @@ public static void closeCatalog() {
}
}
+ @Test
+ void testDatabaseAndTableApiAgainstMetastore() throws Exception {
+ String databaseName = "catalog_api_db";
+ ObjectPath databaseTablePath = new ObjectPath(databaseName,
"catalog_api_table");
+ hoodieCatalog.dropDatabase(databaseName, true, true);
+
+ try {
+ CatalogDatabase database =
+ new CatalogDatabaseImpl(new HashMap<>(), "catalog api database");
+ hoodieCatalog.createDatabase(databaseName, database, false);
+
+ assertTrue(hoodieCatalog.databaseExists(databaseName));
+ assertTrue(hoodieCatalog.listDatabases().contains(databaseName));
+ CatalogDatabase storedDatabase = hoodieCatalog.getDatabase(databaseName);
+ assertEquals("catalog api database", storedDatabase.getComment());
+ assertNotNull(storedDatabase.getProperties().get(DATABASE_LOCATION_URI));
+ assertThrows(
+ DatabaseAlreadyExistException.class,
+ () -> hoodieCatalog.createDatabase(databaseName, database, false));
+ hoodieCatalog.createDatabase(databaseName, database, true);
+
+ Map<String, String> changedProperties = new HashMap<>();
+ changedProperties.put("purpose", "coverage");
+ changedProperties.put("is_generic", "true");
+ hoodieCatalog.alterDatabase(
+ databaseName,
+ new CatalogDatabaseImpl(changedProperties, null),
+ false);
+ assertEquals(
+ "coverage",
+
hoodieCatalog.getDatabase(databaseName).getProperties().get("purpose"));
+ assertFalse(
+
hoodieCatalog.getDatabase(databaseName).getProperties().containsKey("is_generic"));
+
+ String newLocation = new Path(
+ hoodieCatalog.getHiveConf().getVar(
+
org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREWAREHOUSE),
+ databaseName + "_relocated").toString();
+ Map<String, String> locationProperties = new HashMap<>();
+ locationProperties.put(ALTER_DATABASE_OP,
AlterHiveDatabaseOp.CHANGE_LOCATION.name());
+ locationProperties.put(DATABASE_LOCATION_URI, newLocation);
+ hoodieCatalog.alterDatabase(
+ databaseName,
+ new CatalogDatabaseImpl(locationProperties, null),
+ false);
+ assertNotNull(
+
hoodieCatalog.getDatabase(databaseName).getProperties().get(DATABASE_LOCATION_URI));
+
+ Map<String, String> userOwner = new HashMap<>();
+ userOwner.put(ALTER_DATABASE_OP,
AlterHiveDatabaseOp.CHANGE_OWNER.name());
+ userOwner.put(DATABASE_OWNER_NAME, "catalog-user");
+ userOwner.put(DATABASE_OWNER_TYPE, USER_OWNER);
+ hoodieCatalog.alterDatabase(
+ databaseName,
+ new CatalogDatabaseImpl(userOwner, null),
+ false);
+ assertEquals(
+ PrincipalType.USER,
+ hoodieCatalog.getClient().getDatabase(databaseName).getOwnerType());
+
+ Map<String, String> roleOwner = new HashMap<>();
+ roleOwner.put(ALTER_DATABASE_OP,
AlterHiveDatabaseOp.CHANGE_OWNER.name());
+ roleOwner.put(DATABASE_OWNER_NAME, "catalog-role");
+ roleOwner.put(DATABASE_OWNER_TYPE, ROLE_OWNER);
+ hoodieCatalog.alterDatabase(
+ databaseName,
+ new CatalogDatabaseImpl(roleOwner, null),
+ false);
+ assertEquals(
+ PrincipalType.ROLE,
+ hoodieCatalog.getClient().getDatabase(databaseName).getOwnerType());
+
+ Map<String, String> invalidOwner = new HashMap<>();
+ invalidOwner.put(ALTER_DATABASE_OP,
AlterHiveDatabaseOp.CHANGE_OWNER.name());
+ invalidOwner.put(DATABASE_OWNER_NAME, "catalog-group");
+ invalidOwner.put(DATABASE_OWNER_TYPE, "GROUP");
+ assertThrows(
+ org.apache.flink.table.catalog.exceptions.CatalogException.class,
+ () -> hoodieCatalog.alterDatabase(
+ databaseName,
+ new CatalogDatabaseImpl(invalidOwner, null),
+ false));
+
+ hoodieCatalog.alterDatabase(
+ "missing_catalog_api_db",
+ new CatalogDatabaseImpl(Collections.emptyMap(), null),
+ true);
+ assertThrows(
+ DatabaseNotExistException.class,
+ () -> hoodieCatalog.alterDatabase(
+ "missing_catalog_api_db",
+ new CatalogDatabaseImpl(Collections.emptyMap(), null),
+ false));
+
+ Map<String, String> tableOptions = new HashMap<>();
+ tableOptions.put(CONNECTOR.key(), "hudi");
+ CatalogTable catalogTable =
+ CatalogUtils.createCatalogTable(schema, partitions, tableOptions,
"stored in hms");
+ hoodieCatalog.createTable(databaseTablePath, catalogTable, false);
+ assertThrows(
+ TableAlreadyExistException.class,
+ () -> hoodieCatalog.createTable(databaseTablePath, catalogTable,
false));
+ hoodieCatalog.createTable(databaseTablePath, catalogTable, true);
+
+
assertTrue(hoodieCatalog.listTables(databaseName).contains(databaseTablePath.getObjectName()));
+ assertTrue(hoodieCatalog.tableExists(databaseTablePath));
+ Table hiveTable = hoodieCatalog.getHiveTable(databaseTablePath);
+ assertEquals("hudi", hiveTable.getParameters().get(CONNECTOR.key()));
+ assertEquals("stored in hms",
hiveTable.getParameters().get(TableOptionProperties.COMMENT));
+ assertEquals(
+
"uuid:int,name:string,age:int,infos:array<string>,ts_3:timestamp,ts_6:timestamp",
+ hiveTable.getSd().getCols().stream()
+ .filter(field -> !field.getName().startsWith("_hoodie_"))
+ .map(field -> field.getName() + ":" + field.getType())
+ .collect(Collectors.joining(",")));
+ assertEquals(
+ schema.getColumns().stream()
+ .map(Schema.UnresolvedColumn::getName)
+ .collect(Collectors.toList()),
+
hoodieCatalog.getTable(databaseTablePath).getUnresolvedSchema().getColumns().stream()
+ .map(Schema.UnresolvedColumn::getName)
+ .collect(Collectors.toList()));
+
+ String metastoreUris = hoodieCatalog.getHiveConf().getVar(
+ org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS);
+ try {
+ hoodieCatalog.getHiveConf().setVar(
+ org.apache.hadoop.hive.conf.HiveConf.ConfVars.METASTOREURIS,
+ "thrift://localhost:9083");
+ Map<String, String> supplementedOptions =
+ hoodieCatalog.getTable(databaseTablePath).getOptions();
+ assertEquals("true",
supplementedOptions.get(FlinkOptions.HIVE_SYNC_ENABLED.key()));
Review Comment:
Please assert that `HIVE_SYNC_METASTORE_URIS` equals
`thrift://localhost:9083`. Pointing Hive sync back to the catalog's remote
metastore is the operationally important part of this supplementation path, but
every current assertion still passes if that option is missing or stale.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
