prashantwason commented on a change in pull request #5179:
URL: https://github.com/apache/hudi/pull/5179#discussion_r839328936
##########
File path:
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
##########
@@ -498,4 +502,46 @@ public void testBulkInsertRecords(String bulkInsertMode)
throws Exception {
public void testBulkInsertRecordsWithGlobalSort(String bulkInsertMode)
throws Exception {
testBulkInsertRecords(bulkInsertMode);
}
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testPartitionMetafileFormat(boolean
partitionMetafileUseDataFormat) throws Exception {
+ // By default there is no format specified for partition metafile
+ HoodieWriteConfig config = HoodieWriteConfig.newBuilder()
+ .withPath(basePath).withSchema(TRIP_EXAMPLE_SCHEMA).build();
+ HoodieSparkCopyOnWriteTable table = (HoodieSparkCopyOnWriteTable)
HoodieSparkTable.create(config, context, metaClient);
+ assertFalse(table.getPartitionMetafileFormat().isPresent());
+
+ if (partitionMetafileUseDataFormat) {
+ // Add the setting to use datafile format
+ Properties properties = new Properties();
+
properties.setProperty(HoodieTableConfig.PARTITION_METAFILE_USE_DATA_FORMAT.key(),
"true");
+ initMetaClient(HoodieTableType.COPY_ON_WRITE, properties);
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+
assertTrue(metaClient.getTableConfig().getPartitionMetafileFormat().isPresent());
+ table = (HoodieSparkCopyOnWriteTable) HoodieSparkTable.create(config,
context, metaClient);
+ assertTrue(table.getPartitionMetafileFormat().isPresent());
+ }
+
+ String instantTime = makeNewCommitTime();
+ SparkRDDWriteClient writeClient = getHoodieWriteClient(config);
+ writeClient.startCommitWithTime(instantTime);
+
+ // Insert new records
+ final JavaRDD<HoodieRecord> inputRecords =
generateTestRecordsForBulkInsert(jsc, 10);
+ writeClient.bulkInsert(inputRecords, instantTime);
+
+ // Partition metafile should be created
+ Path partitionPath = new Path(basePath,
HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH);
+ assertTrue(HoodiePartitionMetadata.hasPartitionMetadata(fs,
partitionPath));
+ Option<Path> metafilePath =
HoodiePartitionMetadata.getPartitionMetafilePath(fs, partitionPath);
+ if (partitionMetafileUseDataFormat) {
+ // Extension should be the same as the data file format of the table
+
assertTrue(metafilePath.get().toString().endsWith(table.getBaseFileFormat().getFileExtension()));
Review comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
