marton-bod commented on a change in pull request #2123:
URL: https://github.com/apache/iceberg/pull/2123#discussion_r561701108
##########
File path:
mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
##########
@@ -521,4 +438,92 @@ public void
testCreateTableWithNotSupportedTypesWithAutoConversion() {
shell.executeStatement("DROP TABLE not_supported_types");
}
}
+
+ @Test
+ public void testIcebergAndHmsTableProperties() throws TException,
InterruptedException {
+ TableIdentifier identifier = TableIdentifier.of("default", "customers");
+
+ shell.executeStatement(String.format("CREATE EXTERNAL TABLE
default.customers " +
+ "STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler' %s" +
+ "TBLPROPERTIES ('%s'='%s', '%s'='%s', '%s'='%s')",
+ testTables.locationForCreateTableSQL(identifier), // we need the
location for HadoopTable based tests only
+ InputFormatConfig.TABLE_SCHEMA,
SchemaParser.toJson(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA),
+ InputFormatConfig.PARTITION_SPEC, PartitionSpecParser.toJson(SPEC),
+ "custom_property", "initial_val"));
+
+
+ // Check the Iceberg table parameters
+ org.apache.iceberg.Table icebergTable = testTables.loadTable(identifier);
+
+ Map<String, String> expectedIcebergProperties = new HashMap<>();
+ expectedIcebergProperties.put("custom_property", "initial_val");
+ expectedIcebergProperties.put("EXTERNAL", "TRUE");
+ expectedIcebergProperties.put("storage_handler",
HiveIcebergStorageHandler.class.getName());
+ if (Catalogs.hiveCatalog(shell.getHiveConf())) {
+ expectedIcebergProperties.put(TableProperties.ENGINE_HIVE_ENABLED,
"true");
+ }
+ if (MetastoreUtil.hive3PresentOnClasspath()) {
+ expectedIcebergProperties.put("bucketing_version", "2");
+ }
+ Assert.assertEquals(expectedIcebergProperties, icebergTable.properties());
+
+ // Check the HMS table parameters
+ org.apache.hadoop.hive.metastore.api.Table hmsTable =
getHmsTable("default", "customers");
+ Map<String, String> hmsParams = hmsTable.getParameters()
+ .entrySet().stream()
+ .filter(e -> !IGNORED_PARAMS.contains(e.getKey()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+ if (Catalogs.hiveCatalog(shell.getHiveConf())) {
+ Assert.assertEquals(9, hmsParams.size());
+ Assert.assertEquals("initial_val", hmsParams.get("custom_property"));
+ Assert.assertEquals("TRUE",
hmsParams.get(InputFormatConfig.EXTERNAL_TABLE_PURGE));
+ Assert.assertEquals("TRUE", hmsParams.get("EXTERNAL"));
+ Assert.assertEquals("true",
hmsParams.get(TableProperties.ENGINE_HIVE_ENABLED));
+ Assert.assertEquals(HiveIcebergStorageHandler.class.getName(),
+ hmsParams.get(hive_metastoreConstants.META_TABLE_STORAGE));
+
Assert.assertEquals(BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE.toUpperCase(),
+ hmsParams.get(BaseMetastoreTableOperations.TABLE_TYPE_PROP));
+
Assert.assertTrue(hmsParams.get(BaseMetastoreTableOperations.METADATA_LOCATION_PROP)
+ .startsWith(icebergTable.location()));
+
Assert.assertNull(hmsParams.get(BaseMetastoreTableOperations.PREVIOUS_METADATA_LOCATION_PROP));
+ Assert.assertNotNull(hmsParams.get(hive_metastoreConstants.DDL_TIME));
+ } else {
+ Assert.assertEquals(7, hmsParams.size());
+ Assert.assertNull(hmsParams.get(TableProperties.ENGINE_HIVE_ENABLED));
+ }
+
+ // Check HMS inputformat/outputformat/serde
+ Assert.assertEquals(HiveIcebergInputFormat.class.getName(),
hmsTable.getSd().getInputFormat());
+ Assert.assertEquals(HiveIcebergOutputFormat.class.getName(),
hmsTable.getSd().getOutputFormat());
+ Assert.assertEquals(HiveIcebergSerDe.class.getName(),
hmsTable.getSd().getSerdeInfo().getSerializationLib());
+
+ // Add two new properties to the Iceberg table and update an existing one
+ icebergTable.updateProperties()
+ .set("new_prop_1", "true")
+ .set("new_prop_2", "false")
+ .set("custom_property", "new_val")
+ .commit();
+
+ // Refresh the HMS table to see if new Iceberg properties got synced into
HMS
+ hmsParams = getHmsTable("default", "customers").getParameters()
+ .entrySet().stream()
+ .filter(e -> !IGNORED_PARAMS.contains(e.getKey()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+ if (Catalogs.hiveCatalog(shell.getHiveConf())) {
+ Assert.assertEquals(12, hmsParams.size()); // 2 newly-added properties +
previous_metadata_location prop
+ Assert.assertEquals("true", hmsParams.get("new_prop_1"));
+ Assert.assertEquals("false", hmsParams.get("new_prop_2"));
+ Assert.assertEquals("new_val", hmsParams.get("custom_property"));
+
Assert.assertNotNull(hmsParams.get(BaseMetastoreTableOperations.PREVIOUS_METADATA_LOCATION_PROP));
+ } else {
+ Assert.assertEquals(7, hmsParams.size());
+ }
+ }
+
+ private org.apache.hadoop.hive.metastore.api.Table getHmsTable(String
dbName, String tableName)
+ throws TException, InterruptedException {
+ return shell.metastore().clientPool().run(client ->
client.getTable(dbName, tableName));
Review comment:
For now, I've moved the getTable method into TestMetastore because I
think other test classes could make use of this as well
##########
File path:
hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
##########
@@ -191,6 +193,10 @@ public void reset() throws Exception {
}
}
+ public Table getTable(String dbName, String tableName) throws TException,
InterruptedException {
Review comment:
I think so. Keeping the method gives us future flexibility to run custom
metastore client methods without touching the TestHiveMetastore interface, but
at the same time, it feels like we're breaching the TestMetastore abstraction
by exposing its client pool directly, so it's better to introduce new methods
like getTable whenever it becomes necessary.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
