nsivabalan commented on code in PR #7921:
URL: https://github.com/apache/hudi/pull/7921#discussion_r1103501668
##########
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieIndexer.java:
##########
@@ -147,6 +148,26 @@ public void testIndexerWithFilesPartition() {
indexMetadataPartitionsAndAssert(FILES, Collections.emptyList(),
Arrays.asList(new MetadataPartitionType[] {COLUMN_STATS, BLOOM_FILTERS}),
tableName);
}
+ @Test
+ public void testIndexerWithWriter() {
+ // Test the case where the indexer is running, i.e., the delta commit in
the metadata table
+ // is inflight, while the regular writer is updating metadata table.
+ // The delta commit from the indexer should not be rolled back.
+ String tableName = "indexer_with_writer";
+ // enable files and bloom_filters on the regular write client
+ HoodieMetadataConfig.Builder metadataConfigBuilder =
+ getMetadataConfigBuilder(true,
false).withMetadataIndexBloomFilter(true);
+ upsertToTable(metadataConfigBuilder.build(), tableName);
+
+ // validate table config
+
assertTrue(reload(metaClient).getTableConfig().getMetadataPartitions().contains(FILES.getPartitionPath()));
+
assertTrue(reload(metaClient).getTableConfig().getMetadataPartitions().contains(BLOOM_FILTERS.getPartitionPath()));
+
+ scheduleAndExecuteIndexing(COLUMN_STATS, tableName);
+ // metaClient.getActiveTimeline().filter(i -> i.getAction())
Review Comment:
can we remove extraneous code?
##########
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java:
##########
@@ -875,7 +879,11 @@ public void buildMetadataPartitions(HoodieEngineContext
engineContext, List<Hood
inflightIndexes.addAll(indexPartitionInfos.stream().map(HoodieIndexPartitionInfo::getMetadataPartitionPath).collect(Collectors.toSet()));
dataMetaClient.getTableConfig().setValue(HoodieTableConfig.TABLE_METADATA_PARTITIONS_INFLIGHT.key(),
String.join(",", inflightIndexes));
HoodieTableConfig.update(dataMetaClient.getFs(), new
Path(dataMetaClient.getMetaPath()), dataMetaClient.getTableConfig().getProps());
- initialCommit(indexUptoInstantTime, partitionTypes);
+
+ String commitInstantTime =
+
metadataMetaClient.getActiveTimeline().containsInstant(indexUptoInstantTime)
+ ? indexUptoInstantTime + METADATA_INDEXER_TIME_SUFFIX :
indexUptoInstantTime;
Review Comment:
we should not hit the else block. lets do checkState and fail
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
