This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20245-23a122a1f6453ffb29ed03d4eaec346f0bb212ae in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 7888bf909694f3377c2ea7dc860b78e16509623f Author: Vinish Reddy <[email protected]> AuthorDate: Sun Nov 8 01:29:09 2026 +0000 fix(common): reload index definitions before changing them (#20245) --- .../hudi/common/table/HoodieTableMetaClient.java | 20 ++++++++++++ .../common/table/TestHoodieTableMetaClient.java | 37 ++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 87218cae4f40..dd55bacac555 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -252,6 +252,7 @@ public class HoodieTableMetaClient implements Serializable { * @return true if index definition is updated. */ public boolean buildIndexDefinition(HoodieIndexDefinition indexDefinition) { + reloadIndexMetadata(); String indexName = indexDefinition.getIndexName(); boolean updateIndexDefn = true; if (indexMetadataOpt.isPresent()) { @@ -292,6 +293,7 @@ public class HoodieTableMetaClient implements Serializable { * @param indexName Name of the index */ public void deleteIndexDefinition(String indexName) { + reloadIndexMetadata(); checkState(indexMetadataOpt.isPresent(), "Index metadata is not present"); indexMetadataOpt.get().getIndexDefinitions().remove(indexName); writeIndexMetadataToStorage(); @@ -300,6 +302,24 @@ public class HoodieTableMetaClient implements Serializable { } } + /** + * Reloads the index definitions from storage before they are changed. The definitions are written as one file, so a + * change applied to the definitions this meta client loaded earlier would drop the definitions that another meta + * client of the table wrote since, for example the secondary index definition that the metadata writer registers + * during a commit before the commit updates the column stats index definition. + */ + private void reloadIndexMetadata() { + StoragePath indexDefinitionPath = new StoragePath(getIndexDefinitionPath()); + try { + Option<byte[]> bytesOpt = FileIOUtils.readDataFromPath(storage, indexDefinitionPath, true); + if (bytesOpt.isPresent()) { + indexMetadataOpt = Option.of(HoodieIndexMetadata.fromJson(fromUTF8Bytes(bytesOpt.get()))); + } + } catch (IOException e) { + throw new HoodieIOException("Could not load index definition at path: " + indexDefinitionPath, e); + } + } + /** * Writes the current index metadata to storage. */ diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java index ccf8335f6e53..6f300a6b5a04 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java @@ -593,6 +593,43 @@ class TestHoodieTableMetaClient extends HoodieCommonTestHarness { assertTrue(indexMetadata.getIndexDefinitions().isEmpty()); } + @Test + void testIndexDefinitionChangesKeepDefinitionsOfOtherMetaClients() throws IOException { + final String basePath = tempDir.toAbsolutePath() + Path.SEPARATOR + "t8"; + HoodieTableMetaClient firstMetaClient = HoodieTableMetaClient.newTableBuilder() + .setTableType(HoodieTableType.COPY_ON_WRITE.name()) + .setTableName("table") + .initTable(this.metaClient.getStorageConf(), basePath); + // both meta clients load the index definitions before either of them changes them + HoodieTableMetaClient secondMetaClient = HoodieTableMetaClient.reload(firstMetaClient); + String secondaryIndexName = MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() + "c1"; + String columnStatsIndexName = MetadataPartitionType.COLUMN_STATS.getPartitionPath(); + + firstMetaClient.buildIndexDefinition(HoodieIndexDefinition.newBuilder() + .withIndexName(secondaryIndexName) + .withIndexType(MetadataPartitionType.SECONDARY_INDEX.getPartitionPath()) + .withVersion(HoodieIndexVersion.getCurrentVersion(HoodieTableVersion.current(), secondaryIndexName)) + .withSourceFields(Collections.singletonList("c1")) + .build()); + // the second meta client did not load the secondary index definition, and must not drop it + secondMetaClient.buildIndexDefinition(HoodieIndexDefinition.newBuilder() + .withIndexName(columnStatsIndexName) + .withIndexType(columnStatsIndexName) + .withIndexFunction(columnStatsIndexName) + .withVersion(HoodieIndexVersion.getCurrentVersion(HoodieTableVersion.current(), columnStatsIndexName)) + .withSourceFields(Arrays.asList("c1", "c2")) + .build()); + HoodieTableMetaClient reloadedMetaClient = HoodieTableMetaClient.reload(firstMetaClient); + assertTrue(reloadedMetaClient.getIndexForMetadataPartition(secondaryIndexName).isPresent()); + assertTrue(reloadedMetaClient.getIndexForMetadataPartition(columnStatsIndexName).isPresent()); + + // a delete through the first meta client must keep the definition the second meta client wrote + firstMetaClient.deleteIndexDefinition(secondaryIndexName); + reloadedMetaClient = HoodieTableMetaClient.reload(firstMetaClient); + assertFalse(reloadedMetaClient.getIndexForMetadataPartition(secondaryIndexName).isPresent()); + assertTrue(reloadedMetaClient.getIndexForMetadataPartition(columnStatsIndexName).isPresent()); + } + @Test void testReadIndexDefFromStorage() throws Exception { final String basePath = tempDir.toAbsolutePath() + Path.SEPARATOR + "t8";
