This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch release-1.2.1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 4d06094008a7a4d347e80aa88b3f8120850d3407 Author: Lokesh Jain <[email protected]> AuthorDate: Tue May 26 11:27:06 2026 +0530 fix: RLI bootstrap fails due to NPE with cleaner table service (#18836) Co-authored-by: Lokesh Jain <[email protected]> (cherry picked from commit 652d95253648f1d7b5437e803a405cfa9d1f6f86) --- .../metadata/HoodieBackedTableMetadataWriter.java | 24 +++++++++++- .../TestHoodieBackedTableMetadataWriter.java | 45 ++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 5d09762909be..7035c40c9b42 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -58,6 +58,7 @@ import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaCache; import org.apache.hudi.common.schema.HoodieSchemaUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; @@ -934,6 +935,27 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab .sum(); } + /** + * Resolves the data schema (with metadata fields added) for use during record index bootstrap. + * When the write config does not carry a schema (e.g. table-service operations such as clean), + * falls back to resolving the schema from the table's commit history / data files. + */ + static HoodieSchema resolveDataSchemaForRLIBootstrap(HoodieTableMetaClient metaClient, HoodieWriteConfig dataWriteConfig) { + String writeSchemaStr = dataWriteConfig.getWriteSchema(); + HoodieSchema rawSchema; + if (writeSchemaStr != null) { + rawSchema = HoodieSchema.parse(writeSchemaStr); + } else { + try { + rawSchema = new TableSchemaResolver(metaClient).getTableSchema(false); + } catch (Exception e) { + throw new HoodieException( + String.format("Could not resolve schema for table %s for record index bootstrap", metaClient.getBasePath()), e); + } + } + return HoodieSchemaCache.intern(HoodieSchemaUtils.addMetadataFields(rawSchema, dataWriteConfig.allowOperationMetadataField())); + } + /** * Fetch record locations from FileSlice snapshot. * @@ -971,7 +993,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab final FileSlice fileSlice = partitionAndFileSlice.getValue(); final String fileId = fileSlice.getFileId(); HoodieReaderContext<T> readerContext = readerContextFactory.getContext(); - HoodieSchema dataSchema = HoodieSchemaCache.intern(HoodieSchemaUtils.addMetadataFields(HoodieSchema.parse(dataWriteConfig.getWriteSchema()), dataWriteConfig.allowOperationMetadataField())); + HoodieSchema dataSchema = resolveDataSchemaForRLIBootstrap(metaClient, dataWriteConfig); HoodieSchema requestedSchema = metaClient.getTableConfig().populateMetaFields() ? getRecordKeySchema() : HoodieSchemaUtils.projectSchema(dataSchema, Arrays.asList(metaClient.getTableConfig().getRecordKeyFields().orElse(new String[0]))); Option<InternalSchema> internalSchemaOption = SerDeHelper.fromJson(dataWriteConfig.getInternalSchema()); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java index 4065f113d518..7fcc2ce9a5d8 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataWriter.java @@ -25,7 +25,9 @@ import org.apache.hudi.common.data.HoodieData; import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -54,6 +56,7 @@ import java.util.stream.Stream; import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -64,6 +67,7 @@ import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -377,6 +381,47 @@ class TestHoodieBackedTableMetadataWriter { assertDoesNotThrow(() -> validateRollbackMethod.invoke(writer, instantToRollback)); } + // ---- resolveDataSchemaForRLIBootstrap tests ---- + + private static final String SIMPLE_SCHEMA_JSON = + "{\"type\":\"record\",\"name\":\"Test\",\"namespace\":\"test\"," + + "\"fields\":[{\"name\":\"id\",\"type\":\"string\"}]}"; + + @Test + void resolveDataSchemaForRLIBootstrap_usesConfigSchemaWhenPresent() { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class); + when(writeConfig.getWriteSchema()).thenReturn(SIMPLE_SCHEMA_JSON); + when(writeConfig.allowOperationMetadataField()).thenReturn(false); + + HoodieSchema result = HoodieBackedTableMetadataWriter.resolveDataSchemaForRLIBootstrap(metaClient, writeConfig); + + assertNotNull(result); + // metadata fields (_hoodie_*) should have been prepended + assertTrue(result.getFields().stream().anyMatch(f -> f.name().startsWith("_hoodie_"))); + } + + @Test + void resolveDataSchemaForRLIBootstrap_fallsBackToTableSchemaResolverWhenNull() { + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class); + HoodieWriteConfig writeConfig = mock(HoodieWriteConfig.class); + when(writeConfig.getWriteSchema()).thenReturn(null); + when(writeConfig.allowOperationMetadataField()).thenReturn(false); + + HoodieSchema tableSchema = HoodieSchema.parse(SIMPLE_SCHEMA_JSON); + try (org.mockito.MockedConstruction<TableSchemaResolver> mockedResolver = + mockConstruction(TableSchemaResolver.class, + (resolver, ctx) -> when(resolver.getTableSchema(false)).thenReturn(tableSchema))) { + + HoodieSchema result = HoodieBackedTableMetadataWriter.resolveDataSchemaForRLIBootstrap(metaClient, writeConfig); + + assertNotNull(result); + assertTrue(result.getFields().stream().anyMatch(f -> f.name().startsWith("_hoodie_"))); + // exactly one TableSchemaResolver was constructed (with metaClient) + assertEquals(1, mockedResolver.constructed().size()); + } + } + @SuppressWarnings("deprecation") private HoodieActiveTimeline createMockTimeline(List<HoodieInstant> instants) { ActiveTimelineV2 timeline = new ActiveTimelineV2();
