This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 98462b48e15b fix(metadata): correct index definition lookup and
improve mdt read coverage (#19359)
98462b48e15b is described below
commit 98462b48e15b52fcbd47c4bf8bf3a3dda79020ae
Author: Shuo Cheng <[email protected]>
AuthorDate: Tue Jul 28 17:59:12 2026 +0800
fix(metadata): correct index definition lookup and improve mdt read
coverage (#19359)
---
.../hudi/client/TestJavaHoodieBackedMetadata.java | 116 +++++
.../hudi/metadata/MetadataPartitionType.java | 9 +-
.../metadata/TestBaseFileRecordParsingUtils.java | 90 ++++
.../TestHoodieBackedTableMetadataDataCleanup.java | 389 +++++++++++++-
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 570 ++++++++++++++++++++-
.../hudi/metadata/TestMetadataPartitionType.java | 77 +++
.../hudi/metadata/TestBaseTableMetadata.java | 299 +++++++++++
.../TestFileSystemBackedTableMetadata.java | 36 ++
.../hudi/metadata/TestHoodieMetadataPayload.java | 63 +++
9 files changed, 1639 insertions(+), 10 deletions(-)
diff --git
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
index 7bff7b8f2cbe..776816420352 100644
---
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
+++
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
@@ -44,6 +44,7 @@ import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
+import org.apache.hudi.common.model.HoodieRecordGlobalLocation;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.model.TableServiceType;
@@ -76,6 +77,7 @@ import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.JsonUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.common.util.hash.PartitionIndexID;
import org.apache.hudi.config.HoodieArchivalConfig;
import org.apache.hudi.config.HoodieCleanConfig;
@@ -99,6 +101,7 @@ import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataUtil;
import org.apache.hudi.metadata.JavaHoodieBackedTableMetadataWriter;
import org.apache.hudi.metadata.MetadataPartitionType;
+import org.apache.hudi.metadata.RawKey;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -164,6 +167,8 @@ import static
org.apache.hudi.metadata.HoodieTableMetadata.getMetadataTableBaseP
import static
org.apache.hudi.metadata.HoodieTableMetadataUtil.deleteMetadataTable;
import static org.apache.hudi.metadata.MetadataPartitionType.COLUMN_STATS;
import static org.apache.hudi.metadata.MetadataPartitionType.FILES;
+import static org.apache.hudi.metadata.MetadataPartitionType.PARTITION_STATS;
+import static org.apache.hudi.metadata.MetadataPartitionType.RECORD_INDEX;
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -1322,6 +1327,117 @@ public class TestJavaHoodieBackedMetadata extends
TestHoodieMetadataBase {
}
}
+ @Test
+ public void testMetadataReadRoundTrip() throws Exception {
+ this.tableType = COPY_ON_WRITE;
+ initPath();
+ initFileSystem(basePath, storageConf);
+ storage.createDirectory(new StoragePath(basePath));
+ initMetaClient(tableType);
+ initTestDataGenerator();
+ metadataTableBasePath = getMetadataTableBasePath(basePath);
+
+ HoodieJavaEngineContext engineContext = new
HoodieJavaEngineContext(storageConf);
+ HoodieWriteConfig writeConfig = getWriteConfigBuilder(true, true, false)
+ .withMetadataConfig(HoodieMetadataConfig.newBuilder()
+ .enable(true)
+ .withMetadataIndexColumnStats(true)
+ .withMetadataIndexPartitionStats(true)
+
.withColumnStatsIndexForColumns(HoodieRecord.RECORD_KEY_METADATA_FIELD)
+ .withEnableGlobalRecordLevelIndex(true)
+ .withRecordIndexFileGroupCount(3, 3)
+ .build())
+ .build();
+
+ try (HoodieJavaWriteClient client = new
HoodieJavaWriteClient(engineContext, writeConfig)) {
+ String instantTime = client.startCommit();
+ List<HoodieRecord> records = dataGen.generateInserts(instantTime, 30);
+ Map<String, String> recordKeyToPartition = records.stream()
+ .collect(Collectors.toMap(HoodieRecord::getRecordKey,
HoodieRecord::getPartitionPath));
+ List<WriteStatus> writeStatuses = client.insert(records, instantTime);
+ client.commit(instantTime, writeStatuses);
+ assertNoWriteErrors(writeStatuses);
+
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+
assertTrue(metaClient.getTableConfig().isMetadataPartitionAvailable(FILES));
+
assertTrue(metaClient.getTableConfig().isMetadataPartitionAvailable(COLUMN_STATS));
+
assertTrue(metaClient.getTableConfig().isMetadataPartitionAvailable(PARTITION_STATS));
+
assertTrue(metaClient.getTableConfig().isMetadataPartitionAvailable(RECORD_INDEX));
+
+ try (HoodieTableMetadata tableMetadata = metadata(client);
+ HoodieTableMetadata fileSystemMetadata = new
FileSystemBackedTableMetadata(
+ engineContext, metaClient.getTableConfig(),
metaClient.getStorage(), basePath)) {
+ List<String> expectedPartitions =
fileSystemMetadata.getAllPartitionPaths();
+ List<String> actualPartitions = tableMetadata.getAllPartitionPaths();
+ Collections.sort(expectedPartitions);
+ Collections.sort(actualPartitions);
+ assertEquals(expectedPartitions, actualPartitions);
+ assertFalse(actualPartitions.isEmpty());
+
+ List<Pair<String, String>> partitionAndFileNames = new ArrayList<>();
+ for (String partition : actualPartitions) {
+ StoragePath partitionPath = partition.isEmpty()
+ ? new StoragePath(basePath)
+ : new StoragePath(basePath, partition);
+ List<String> expectedFileNames =
fileSystemMetadata.getAllFilesInPartition(partitionPath).stream()
+ .map(pathInfo -> pathInfo.getPath().getName())
+ .sorted()
+ .collect(Collectors.toList());
+ List<String> actualFileNames =
tableMetadata.getAllFilesInPartition(partitionPath).stream()
+ .map(pathInfo -> pathInfo.getPath().getName())
+ .sorted()
+ .collect(Collectors.toList());
+ assertEquals(expectedFileNames, actualFileNames);
+ assertFalse(actualFileNames.isEmpty());
+ actualFileNames.forEach(fileName ->
partitionAndFileNames.add(Pair.of(partition, fileName)));
+ }
+
+ Map<Pair<String, String>, HoodieMetadataColumnStats> columnStats =
+ tableMetadata.getColumnStats(partitionAndFileNames,
HoodieRecord.RECORD_KEY_METADATA_FIELD);
+ assertEquals(partitionAndFileNames.size(), columnStats.size());
+ columnStats.values().forEach(stats -> {
+ assertEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD,
stats.getColumnName().toString());
+ assertFalse(stats.getIsDeleted());
+ assertNotNull(stats.getMinValue());
+ assertNotNull(stats.getMaxValue());
+ });
+
+ Map<String, String> partitionStatsKeyToPartition =
actualPartitions.stream()
+ .collect(Collectors.toMap(
+ partition -> HoodieTableMetadataUtil.getPartitionStatsIndexKey(
+ partition, HoodieRecord.RECORD_KEY_METADATA_FIELD),
+ partition -> partition));
+ List<RawKey> partitionStatsKeys =
partitionStatsKeyToPartition.keySet().stream()
+ .map(key -> (RawKey) () -> key)
+ .collect(Collectors.toList());
+ List<HoodieRecord<HoodieMetadataPayload>> partitionStats =
tableMetadata.getRecordsByKeyPrefixes(
+ HoodieListData.eager(partitionStatsKeys),
PARTITION_STATS.getPartitionPath(), true).collectAsList();
+ assertEquals(actualPartitions.size(), partitionStats.size());
+ partitionStats.forEach(record -> {
+
assertTrue(partitionStatsKeyToPartition.containsKey(record.getRecordKey()));
+ assertTrue(record.getData().getColumnStatMetadata().isPresent());
+ HoodieMetadataColumnStats stats =
record.getData().getColumnStatMetadata().get();
+
assertEquals(partitionStatsKeyToPartition.get(record.getRecordKey()),
stats.getFileName().toString());
+ assertEquals(HoodieRecord.RECORD_KEY_METADATA_FIELD,
stats.getColumnName().toString());
+ assertFalse(stats.getIsDeleted());
+ assertNotNull(stats.getMinValue());
+ assertNotNull(stats.getMaxValue());
+ });
+
+ List<Pair<String, HoodieRecordGlobalLocation>> recordLocations =
tableMetadata
+ .readRecordIndexLocationsWithKeys(HoodieListData.eager(new
ArrayList<>(recordKeyToPartition.keySet())))
+ .collectAsList();
+ assertEquals(records.size(), recordLocations.size());
+ assertEquals(recordKeyToPartition.keySet(),
+
recordLocations.stream().map(Pair::getLeft).collect(Collectors.toSet()));
+ recordLocations.forEach(entry -> {
+ assertEquals(recordKeyToPartition.get(entry.getLeft()),
entry.getRight().getPartitionPath());
+ assertNotNull(entry.getRight().getFileId());
+ });
+ }
+ }
+ }
+
@Test
public void
testReadRecordIndexLocationsByBucketIdFailsWhenRecordIndexDisabled() throws
Exception {
init(HoodieTableType.COPY_ON_WRITE);
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java
index 179bcc5f648f..6311a7cf1580 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java
@@ -510,7 +510,7 @@ public enum MetadataPartitionType {
return false;
}
// check the index definition already exists or not for this column
- List<HoodieIndexDefinition> indexDefinitions =
getIndexDefinitions(secondaryIndexColumn, PARTITION_NAME_SECONDARY_INDEX,
dataMetaClient);
+ List<HoodieIndexDefinition> indexDefinitions =
getIndexDefinitions(PARTITION_NAME_SECONDARY_INDEX, secondaryIndexColumn,
dataMetaClient);
return indexDefinitions.isEmpty();
}
@@ -531,7 +531,7 @@ public enum MetadataPartitionType {
// get all index definitions for this column and index type
// check if none of the index definitions has index function matching the
expression
- List<HoodieIndexDefinition> indexDefinitions =
getIndexDefinitions(expressionIndexColumn, PARTITION_NAME_EXPRESSION_INDEX,
dataMetaClient);
+ List<HoodieIndexDefinition> indexDefinitions =
getIndexDefinitions(PARTITION_NAME_EXPRESSION_INDEX, expressionIndexColumn,
dataMetaClient);
return indexDefinitions.isEmpty()
|| indexDefinitions.stream().noneMatch(indexDefinition ->
indexDefinition.getIndexFunction().equals(expressionIndexOptions.get(HoodieExpressionIndex.EXPRESSION_OPTION)));
}
@@ -549,11 +549,6 @@ public enum MetadataPartitionType {
return indexDefinitions;
}
- private static boolean isIndexDefinitionPresentForColumn(String
indexedColumn, String indexType, HoodieTableMetaClient dataMetaClient) {
- return dataMetaClient.getIndexMetadata().isPresent() &&
dataMetaClient.getIndexMetadata().get().getIndexDefinitions().values().stream()
- .anyMatch(indexDefinition ->
indexDefinition.getSourceFields().contains(indexedColumn) &&
indexDefinition.getIndexType().equals(indexType));
- }
-
@Override
public String toString() {
return "Metadata partition {"
diff --git
a/hudi-common/src/test/java/org/apache/hudi/metadata/TestBaseFileRecordParsingUtils.java
b/hudi-common/src/test/java/org/apache/hudi/metadata/TestBaseFileRecordParsingUtils.java
new file mode 100644
index 000000000000..4470245ade36
--- /dev/null
+++
b/hudi-common/src/test/java/org/apache/hudi/metadata/TestBaseFileRecordParsingUtils.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metadata;
+
+import org.apache.hudi.common.model.HoodieFileFormat;
+import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.util.FileFormatUtils;
+import org.apache.hudi.core.io.storage.HoodieIOFactory;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StoragePath;
+
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import static
org.apache.hudi.metadata.BaseFileRecordParsingUtils.RecordStatus.DELETE;
+import static
org.apache.hudi.metadata.BaseFileRecordParsingUtils.RecordStatus.INSERT;
+import static
org.apache.hudi.metadata.BaseFileRecordParsingUtils.RecordStatus.UPDATE;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
+
+class TestBaseFileRecordParsingUtils {
+
+ @Test
+ void testRecordKeyStatusClassificationAndSecondaryIndexKeys() {
+ HoodieStorage storage = mock(HoodieStorage.class);
+ HoodieIOFactory ioFactory = mock(HoodieIOFactory.class);
+ FileFormatUtils fileFormatUtils = mock(FileFormatUtils.class);
+
when(ioFactory.getFileFormatUtils(HoodieFileFormat.PARQUET)).thenReturn(fileFormatUtils);
+ when(fileFormatUtils.readRowKeys(any(),
any(StoragePath.class))).thenAnswer(invocation -> {
+ StoragePath path = invocation.getArgument(1);
+ return path.getName().equals("latest.parquet")
+ ? new HashSet<>(Arrays.asList("inserted", "updated"))
+ : new HashSet<>(Arrays.asList("updated", "deleted"));
+ });
+
+ try (MockedStatic<HoodieIOFactory> ioFactoryMock =
mockStatic(HoodieIOFactory.class)) {
+ ioFactoryMock.when(() ->
HoodieIOFactory.getIOFactory(storage)).thenReturn(ioFactory);
+
+ Map<BaseFileRecordParsingUtils.RecordStatus, List<String>> statuses =
+ BaseFileRecordParsingUtils.getRecordKeyStatuses(
+ "/table", "partition", "latest.parquet", "previous.parquet",
storage,
+ EnumSet.allOf(BaseFileRecordParsingUtils.RecordStatus.class));
+ assertEquals(Collections.singletonList("inserted"),
statuses.get(INSERT));
+ assertEquals(Collections.singletonList("updated"), statuses.get(UPDATE));
+ assertEquals(Collections.singletonList("deleted"), statuses.get(DELETE));
+
+ assertTrue(BaseFileRecordParsingUtils.getRecordKeyStatuses(
+ "/table", "partition", "latest.parquet", null, storage,
EnumSet.of(UPDATE, DELETE)).isEmpty());
+ assertEquals(
+ new HashSet<>(Arrays.asList("inserted", "updated")),
+ new HashSet<>(BaseFileRecordParsingUtils.getRecordKeyStatuses(
+ "/table", "partition", "latest.parquet", null, storage,
EnumSet.of(INSERT)).get(INSERT)));
+
+ HoodieWriteStat writeStat = mock(HoodieWriteStat.class);
+ when(writeStat.getPath()).thenReturn("partition/latest.parquet");
+ when(writeStat.getPartitionPath()).thenReturn("partition");
+ when(writeStat.getPrevBaseFile()).thenReturn("previous.parquet");
+ List<String> changedKeys =
+ BaseFileRecordParsingUtils.getRecordKeysDeletedOrUpdated("/table",
writeStat, storage);
+ assertEquals(new HashSet<>(Arrays.asList("updated", "deleted")), new
HashSet<>(changedKeys));
+ }
+ }
+}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java
b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java
index 03d013f4c586..c6c97c66bb7e 100644
---
a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java
+++
b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieBackedTableMetadataDataCleanup.java
@@ -18,23 +18,56 @@
package org.apache.hudi.metadata;
+import org.apache.hudi.avro.model.HoodieMetadataRecord;
+import org.apache.hudi.common.config.HoodieConfig;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.data.HoodieData;
import org.apache.hudi.common.data.HoodieListData;
import org.apache.hudi.common.data.HoodiePairData;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.expression.Expression;
+import org.apache.hudi.common.function.SerializableFunction;
+import org.apache.hudi.common.function.SerializableFunctionUnchecked;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieBaseFile;
+import org.apache.hudi.common.model.HoodieFileFormat;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.schema.internal.Types;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.read.HoodieFileGroupReader;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.ClosableIterator;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.core.io.storage.HoodieFileReaderFactory;
+import org.apache.hudi.core.io.storage.HoodieIOFactory;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.avro.generic.IndexedRecord;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.MockedStatic;
+import java.io.IOException;
import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
@@ -239,4 +272,358 @@ public class TestHoodieBackedTableMetadataDataCleanup {
// Verify cleanup manager was called
verify(mockCleanupManager).ensureDataCleanupOnException(any());
}
-}
\ No newline at end of file
+
+ @Test
+ public void testSecondaryIndexUnsupportedVersion() {
+ HoodieData<String> keys =
HoodieListData.eager(Collections.singletonList("key"));
+ HoodieIndexVersion unsupportedVersion = mock(HoodieIndexVersion.class);
+ try (MockedStatic<HoodieTableMetadataUtil> mockedUtil =
mockStatic(HoodieTableMetadataUtil.class)) {
+ mockedUtil.when(() ->
HoodieTableMetadataUtil.existingIndexVersionOrDefault(anyString(), any()))
+ .thenReturn(unsupportedVersion);
+
+ when(mockMetadata.readSecondaryIndexLocationsWithKeys(keys,
"secondary_index_test")).thenCallRealMethod();
+ when(mockMetadata.readSecondaryIndexLocations(keys,
"secondary_index_test")).thenCallRealMethod();
+ when(mockMetadata.readSecondaryIndexDataTableRecordKeysWithKeys(keys,
"secondary_index_test")).thenCallRealMethod();
+
+ assertThrows(IllegalArgumentException.class,
+ () -> mockMetadata.readSecondaryIndexLocationsWithKeys(keys,
"secondary_index_test"));
+ assertThrows(IllegalArgumentException.class,
+ () -> mockMetadata.readSecondaryIndexLocations(keys,
"secondary_index_test"));
+ assertThrows(IllegalArgumentException.class,
+ () ->
mockMetadata.readSecondaryIndexDataTableRecordKeysWithKeys(keys,
"secondary_index_test"));
+ }
+ }
+
+ @Test
+ public void testSecondaryIndexEmptyAndMissingPartition() {
+ HoodieData<String> emptyKeys =
HoodieListData.eager(Collections.emptyList());
+ String partitionName = "secondary_index_test";
+ when(mockMetadata.readSecondaryIndexDataTableRecordKeysWithKeys(emptyKeys,
partitionName)).thenCallRealMethod();
+
+ try (MockedStatic<HoodieTableMetadataUtil> mockedUtil =
mockStatic(HoodieTableMetadataUtil.class)) {
+ mockedUtil.when(() ->
HoodieTableMetadataUtil.existingIndexVersionOrDefault(anyString(), any()))
+ .thenReturn(HoodieIndexVersion.V1);
+
assertTrue(mockMetadata.readSecondaryIndexDataTableRecordKeysWithKeys(emptyKeys,
partitionName)
+ .collectAsList().isEmpty());
+
+ mockedUtil.when(() ->
HoodieTableMetadataUtil.existingIndexVersionOrDefault(anyString(), any()))
+ .thenReturn(HoodieIndexVersion.V2);
+
assertTrue(mockMetadata.readSecondaryIndexDataTableRecordKeysWithKeys(emptyKeys,
partitionName)
+ .collectAsList().isEmpty());
+
+ HoodieData<String> keys =
HoodieListData.eager(Collections.singletonList("key"));
+ mockedUtil.when(() ->
HoodieTableMetadataUtil.existingIndexVersionOrDefault(anyString(), any()))
+ .thenReturn(HoodieIndexVersion.V1);
+
when(mockTableConfig.getMetadataPartitions()).thenReturn(Collections.emptySet());
+ when(mockMetadata.readSecondaryIndexLocationsWithKeys(keys,
partitionName)).thenCallRealMethod();
+ assertThrows(IllegalStateException.class,
+ () -> mockMetadata.readSecondaryIndexLocationsWithKeys(keys,
partitionName));
+ }
+ }
+
+ @Test
+ public void testEmptyRecordIndexAndMetadataTimeline() throws Exception {
+ Field fileSliceMapField =
HoodieBackedTableMetadata.class.getDeclaredField("partitionFileSliceMap");
+ fileSliceMapField.setAccessible(true);
+ Map<String, List<FileSlice>> fileSliceMap = new HashMap<>();
+ fileSliceMap.put(MetadataPartitionType.RECORD_INDEX.getPartitionPath(),
Collections.emptyList());
+ fileSliceMapField.set(mockMetadata, fileSliceMap);
+
+ when(mockMetadata.readRecordIndexLocations(
+
org.mockito.ArgumentMatchers.<SerializableFunctionUnchecked<List<FileSlice>,
List<FileSlice>>>any()))
+ .thenCallRealMethod();
+ assertTrue(mockMetadata.readRecordIndexLocations(slices ->
slices).collectAsList().isEmpty());
+
+ when(mockMetadata.getSyncedInstantTime()).thenCallRealMethod();
+ when(mockMetadata.getLatestCompactionTime()).thenCallRealMethod();
+ assertFalse(mockMetadata.getSyncedInstantTime().isPresent());
+ assertFalse(mockMetadata.getLatestCompactionTime().isPresent());
+ }
+
+ @Test
+ public void testPartitionFilterFallbackAndBucketValidation() throws
Exception {
+ List<String> selectedPartitions = Arrays.asList("year=2025", "year=2026");
+ Expression expression = mock(Expression.class);
+ when(expression.accept(any())).thenReturn(expression);
+
when(mockMetadata.getPartitionPathWithPathPrefixes(any())).thenReturn(selectedPartitions);
+
when(mockMetadata.getPartitionPathWithPathPrefixUsingFilterExpression(any(),
any(), any()))
+ .thenCallRealMethod();
+
+ assertEquals(selectedPartitions,
mockMetadata.getPartitionPathWithPathPrefixUsingFilterExpression(
+ Collections.singletonList("year="), mock(Types.RecordType.class),
expression));
+
+ Field partitionedMapField =
+
HoodieBackedTableMetadata.class.getDeclaredField("partitionedRLIFileSliceMap");
+ partitionedMapField.setAccessible(true);
+ partitionedMapField.set(mockMetadata, new HashMap<>());
+
when(mockMetadata.getBucketizedFileGroupsForPartitionedRLI(any())).thenCallRealMethod();
+ assertThrows(IllegalArgumentException.class,
+ () ->
mockMetadata.getBucketizedFileGroupsForPartitionedRLI(MetadataPartitionType.FILES));
+
+
when(mockMetadata.getFilegroupsForPartition(MetadataPartitionType.RECORD_INDEX))
+ .thenReturn(Collections.emptyList());
+ assertTrue(mockMetadata.getBucketizedFileGroupsForPartitionedRLI(
+ MetadataPartitionType.RECORD_INDEX).isEmpty());
+
+ FileSlice nonPartitionedSlice = mock(FileSlice.class);
+ when(nonPartitionedSlice.getFileId()).thenReturn("record-index-0000");
+
when(mockMetadata.getFilegroupsForPartition(MetadataPartitionType.RECORD_INDEX))
+ .thenReturn(Collections.singletonList(nonPartitionedSlice));
+ assertThrows(IllegalArgumentException.class,
+ () -> mockMetadata.getBucketizedFileGroupsForPartitionedRLI(
+ MetadataPartitionType.RECORD_INDEX));
+ }
+
+ @Test
+ public void testPartitionedRecordIndexLookupGuards() throws Exception {
+ Method lookupMethod = HoodieBackedTableMetadata.class.getDeclaredMethod(
+ "lookupIndexRecords", HoodieData.class, String.class, List.class,
Option.class);
+ lookupMethod.setAccessible(true);
+ FileSlice partitionedSlice = mock(FileSlice.class);
+
when(partitionedSlice.getFileId()).thenReturn("record-index-partition-x-0000");
+ List<FileSlice> slices = Collections.singletonList(partitionedSlice);
+
+ HoodieData<?> emptyResult = (HoodieData<?>) lookupMethod.invoke(
+ mockMetadata,
+ HoodieListData.eager(Collections.emptyList()),
+ MetadataPartitionType.RECORD_INDEX.getPartitionPath(),
+ slices,
+ Option.empty());
+ assertTrue(emptyResult.collectAsList().isEmpty());
+
+ InvocationTargetException exception = assertThrows(
+ InvocationTargetException.class,
+ () -> lookupMethod.invoke(
+ mockMetadata,
+ HoodieListData.eager(Collections.singletonList("key")),
+ MetadataPartitionType.RECORD_INDEX.getPartitionPath(),
+ slices,
+ Option.empty()));
+ assertTrue(exception.getCause() instanceof IllegalArgumentException);
+ }
+
+ @Test
+ public void testSecondaryIndexEmptyIteratorPath() throws Exception {
+ Method method = HoodieBackedTableMetadata.class.getDeclaredMethod(
+ "readSliceAndFilterByKeys", String.class, List.class, FileSlice.class);
+ method.setAccessible(true);
+ Object iterator = method.invoke(
+ mockMetadata,
+ MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() + "test",
+ Collections.emptyList(),
+ mock(FileSlice.class));
+ assertFalse(((org.apache.hudi.common.util.collection.ClosableIterator<?>)
iterator).hasNext());
+ }
+
+ @Test
+ public void testInitializationFailureDisablesMetadata() throws Exception {
+ Field initializedField =
BaseTableMetadata.class.getDeclaredField("isMetadataTableInitialized");
+ initializedField.setAccessible(true);
+ initializedField.set(mockMetadata, true);
+ Field metadataBasePathField =
+ HoodieBackedTableMetadata.class.getDeclaredField("metadataBasePath");
+ metadataBasePathField.setAccessible(true);
+ metadataBasePathField.set(mockMetadata, "/table/.hoodie/metadata");
+ when(mockMetadata.getStorage()).thenReturn(mock(HoodieStorage.class));
+
+ HoodieTableMetaClient.Builder builder =
mock(HoodieTableMetaClient.Builder.class);
+ when(builder.setStorage(any())).thenReturn(builder);
+ when(builder.setBasePath(anyString())).thenReturn(builder);
+ when(builder.build()).thenThrow(new HoodieException("initialization
failed"));
+
+ try (MockedStatic<HoodieTableMetaClient> metaClientStatic =
+ mockStatic(HoodieTableMetaClient.class)) {
+
metaClientStatic.when(HoodieTableMetaClient::builder).thenReturn(builder);
+ Method initMethod =
HoodieBackedTableMetadata.class.getDeclaredMethod("initIfNeeded");
+ initMethod.setAccessible(true);
+ initMethod.invoke(mockMetadata);
+ }
+
+ assertFalse(mockMetadata.isMetadataTableInitialized());
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ public void testSecondaryIndexRecordMapping() throws Exception {
+ prepareFileSliceRead(false);
+ HoodieRecord<HoodieMetadataPayload> metadataRecord =
+ HoodieMetadataPayload.createPartitionFilesRecord(
+ "key", Collections.emptyMap(), Collections.emptyList());
+ IndexedRecord indexedRecord =
+ (IndexedRecord) metadataRecord.getData().getInsertValue(
+ HoodieMetadataRecord.getClassSchema()).get();
+
+ HoodieFileGroupReader.HoodieFileGroupReaderBuilder builder =
+ mock(HoodieFileGroupReader.HoodieFileGroupReaderBuilder.class);
+ HoodieFileGroupReader fileGroupReader = mock(HoodieFileGroupReader.class);
+ when(builder.withReaderContext(any())).thenReturn(builder);
+ when(builder.withHoodieTableMetaClient(any())).thenReturn(builder);
+ when(builder.withLatestCommitTime(anyString())).thenReturn(builder);
+ when(builder.withBaseFileOption(any())).thenReturn(builder);
+ when(builder.withLogFiles(any())).thenReturn(builder);
+ when(builder.withPartitionPath(anyString())).thenReturn(builder);
+ when(builder.withDataSchema(any())).thenReturn(builder);
+ when(builder.withRequestedSchema(any())).thenReturn(builder);
+ when(builder.withProps(any())).thenReturn(builder);
+ when(builder.withRecordBufferLoader(any())).thenReturn(builder);
+ when(builder.build()).thenReturn(fileGroupReader);
+ when(fileGroupReader.getClosableIterator()).thenReturn(
+
ClosableIterator.wrap(Collections.singletonList(indexedRecord).iterator()));
+
+ FileSlice fileSlice = mock(FileSlice.class);
+ when(fileSlice.getPartitionPath()).thenReturn(
+ MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() + "test");
+ when(fileSlice.getBaseFile()).thenReturn(Option.empty());
+ when(fileSlice.getLogFiles()).thenReturn(Stream.empty());
+
+ try (MockedStatic<HoodieFileGroupReader> readerStatic =
+ mockStatic(HoodieFileGroupReader.class)) {
+ readerStatic.when(HoodieFileGroupReader::builder).thenReturn(builder);
+ Method method = HoodieBackedTableMetadata.class.getDeclaredMethod(
+ "readSliceAndFilterByKeys", String.class, List.class,
FileSlice.class);
+ method.setAccessible(true);
+ ClosableIterator<Pair<String, HoodieRecord<HoodieMetadataPayload>>>
iterator =
+ (ClosableIterator<Pair<String,
HoodieRecord<HoodieMetadataPayload>>>) method.invoke(
+ mockMetadata,
+ MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() +
"test",
+ Collections.singletonList("key"),
+ fileSlice);
+
+ assertTrue(iterator.hasNext());
+ assertEquals("key", iterator.next().getLeft());
+ assertFalse(iterator.hasNext());
+ iterator.close();
+
+ when(fileGroupReader.getClosableIterator())
+ .thenThrow(new IOException("iterator failed"));
+ InvocationTargetException exception = assertThrows(
+ InvocationTargetException.class,
+ () -> method.invoke(
+ mockMetadata,
+ MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() +
"test",
+ Collections.singletonList("key"),
+ fileSlice));
+ assertTrue(exception.getCause() instanceof
org.apache.hudi.exception.HoodieIOException);
+
+ Method scanMethod = HoodieBackedTableMetadata.class.getDeclaredMethod(
+ "scanRecordsItr", FileSlice.class,
SerializableFunctionUnchecked.class);
+ scanMethod.setAccessible(true);
+ InvocationTargetException scanException = assertThrows(
+ InvocationTargetException.class,
+ () -> scanMethod.invoke(
+ mockMetadata,
+ fileSlice,
+
(SerializableFunctionUnchecked<org.apache.avro.generic.GenericRecord,
+ HoodieRecord<HoodieMetadataPayload>>) record -> null));
+ assertTrue(scanException.getCause() instanceof
org.apache.hudi.exception.HoodieIOException);
+ }
+ }
+
+ @Test
+ public void testReusableReaderIOExceptionIsWrapped() throws Exception {
+ prepareFileSliceRead(true);
+ HoodieStorage storage = mock(HoodieStorage.class);
+ when(mockMetadata.getStorage()).thenReturn(storage);
+
+ HoodieBaseFile baseFile = mock(HoodieBaseFile.class);
+ when(baseFile.getPathInfo()).thenReturn(mock(StoragePathInfo.class));
+ FileSlice fileSlice = mock(FileSlice.class);
+
when(fileSlice.getPartitionPath()).thenReturn(MetadataPartitionType.FILES.getPartitionPath());
+ when(fileSlice.getFileGroupId()).thenReturn(
+ new org.apache.hudi.common.model.HoodieFileGroupId(
+ MetadataPartitionType.FILES.getPartitionPath(), "file-id"));
+ when(fileSlice.getBaseFile()).thenReturn(Option.of(baseFile));
+
+ HoodieIOFactory ioFactory = mock(HoodieIOFactory.class);
+ HoodieFileReaderFactory readerFactory =
mock(HoodieFileReaderFactory.class);
+ when(ioFactory.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO))
+ .thenReturn(readerFactory);
+ when(readerFactory.getFileReader(
+ any(HoodieConfig.class),
+ any(StoragePathInfo.class),
+ any(HoodieFileFormat.class),
+ any(Option.class)))
+ .thenThrow(new IOException("reader failed"));
+
+ try (MockedStatic<HoodieIOFactory> ioFactoryStatic =
+ mockStatic(HoodieIOFactory.class)) {
+ ioFactoryStatic.when(() ->
HoodieIOFactory.getIOFactory(storage)).thenReturn(ioFactory);
+ Method method = HoodieBackedTableMetadata.class.getDeclaredMethod(
+ "readSliceWithFilter",
+ org.apache.hudi.common.expression.Predicate.class,
+ FileSlice.class);
+ method.setAccessible(true);
+ InvocationTargetException exception = assertThrows(
+ InvocationTargetException.class,
+ () -> method.invoke(
+ mockMetadata,
+ mock(org.apache.hudi.common.expression.Predicate.class),
+ fileSlice));
+ assertTrue(exception.getCause() instanceof
org.apache.hudi.exception.HoodieIOException);
+ }
+ }
+
+ @Test
+ public void testEmptyShardReturnsEmptyIterator() throws Exception {
+ HoodieEngineContext engineContext = mock(HoodieEngineContext.class);
+ HoodieData<String> emptyKeys =
HoodieListData.eager(Collections.emptyList());
+ HoodieData<HoodieRecord<HoodieMetadataPayload>> emptyResult =
+ HoodieListData.eager(Collections.emptyList());
+ when(mockMetadata.getEngineContext()).thenReturn(engineContext);
+ when(engineContext.parallelize(
+ any(List.class),
org.mockito.ArgumentMatchers.eq(1))).thenReturn(emptyKeys);
+ when(engineContext.mapGroupsByKey(any(), any(), any(),
org.mockito.ArgumentMatchers.eq(true)))
+ .thenAnswer(invocation -> {
+ SerializableFunction<java.util.Iterator<String>,
+ java.util.Iterator<HoodieRecord<HoodieMetadataPayload>>>
processFunction =
+ invocation.getArgument(1);
+
assertFalse(processFunction.apply(Collections.emptyIterator()).hasNext());
+ return emptyResult;
+ });
+
+ Method method = HoodieBackedTableMetadata.class.getDeclaredMethod(
+ "lookupIndexRecords", HoodieData.class, String.class, List.class,
+ Option.class);
+ method.setAccessible(true);
+ Object result = method.invoke(
+ mockMetadata,
+ emptyKeys,
+ MetadataPartitionType.FILES.getPartitionPath(),
+ Arrays.asList(mock(FileSlice.class), mock(FileSlice.class)),
+ Option.empty());
+
+ assertEquals(emptyResult, result);
+ }
+
+ private void prepareFileSliceRead(boolean reuse) throws Exception {
+ HoodieTableMetaClient metadataMetaClient =
mock(HoodieTableMetaClient.class);
+ HoodieActiveTimeline timeline = mock(HoodieActiveTimeline.class);
+ when(metadataMetaClient.getActiveTimeline()).thenReturn(timeline);
+ when(timeline.filterCompletedInstants()).thenReturn(timeline);
+ when(timeline.lastInstant()).thenReturn(Option.empty());
+ HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(tableConfig.populateMetaFields()).thenReturn(true);
+ when(tableConfig.getBaseFileFormat()).thenReturn(HoodieFileFormat.PARQUET);
+ when(metadataMetaClient.getTableConfig()).thenReturn(tableConfig);
+
+ Field metadataMetaClientField =
+ HoodieBackedTableMetadata.class.getDeclaredField("metadataMetaClient");
+ metadataMetaClientField.setAccessible(true);
+ metadataMetaClientField.set(mockMetadata, metadataMetaClient);
+ Field validInstantsField =
+
HoodieBackedTableMetadata.class.getDeclaredField("validInstantTimestamps");
+ validInstantsField.setAccessible(true);
+ validInstantsField.set(mockMetadata, Collections.singleton("001"));
+ Field reuseField =
HoodieBackedTableMetadata.class.getDeclaredField("reuse");
+ reuseField.setAccessible(true);
+ reuseField.set(mockMetadata, reuse);
+ Field metadataConfigField =
BaseTableMetadata.class.getDeclaredField("metadataConfig");
+ metadataConfigField.setAccessible(true);
+ metadataConfigField.set(
+ mockMetadata, HoodieMetadataConfig.newBuilder().enable(true).build());
+ Field storageConfField =
+ AbstractHoodieTableMetadata.class.getDeclaredField("storageConf");
+ storageConfField.setAccessible(true);
+ storageConfField.set(mockMetadata, mock(StorageConfiguration.class));
+ }
+}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
index 77bc52a18d58..5b4f921936da 100644
---
a/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
+++
b/hudi-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
@@ -18,26 +18,67 @@
package org.apache.hudi.metadata;
+import org.apache.hudi.avro.model.HoodieInstantInfo;
+import org.apache.hudi.avro.model.HoodieMetadataColumnStats;
import org.apache.hudi.avro.model.HoodieMetadataRecord;
+import org.apache.hudi.avro.model.HoodieRollbackPlan;
import org.apache.hudi.common.avro.HoodieAvroUtils;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.data.HoodieData;
+import org.apache.hudi.common.data.HoodieListData;
+import org.apache.hudi.common.data.HoodiePairData;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.engine.HoodieLocalEngineContext;
+import org.apache.hudi.common.engine.HoodieReaderContext;
+import org.apache.hudi.common.engine.ReaderContextFactory;
import org.apache.hudi.common.function.SerializableBiFunction;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieIndexDefinition;
import org.apache.hudi.common.model.HoodieIndexMetadata;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
import org.apache.hudi.common.model.HoodieRecordGlobalLocation;
+import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaField;
import org.apache.hudi.common.schema.HoodieSchemaType;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTableVersion;
+import org.apache.hudi.common.table.TableSchemaResolver;
+import org.apache.hudi.common.table.read.HoodieFileGroupReader;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.InstantGenerator;
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
+import org.apache.hudi.common.util.HoodieStorageUtils;
import org.apache.hudi.common.util.Option;
-
+import org.apache.hudi.common.util.collection.ClosableIterator;
+import org.apache.hudi.common.util.collection.ExternalSpillableMap;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieMetadataException;
+import org.apache.hudi.exception.HoodieNotSupportedException;
+import org.apache.hudi.metadata.model.FileInfoAndPartition;
+import org.apache.hudi.metadata.stats.HoodieColumnRangeMetadata;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.StoragePath;
+
+import org.apache.avro.AvroTypeException;
+import org.apache.avro.LogicalTypes;
import org.apache.avro.generic.GenericRecord;
import org.junit.jupiter.api.Test;
-
+import org.mockito.MockedConstruction;
+import org.mockito.MockedStatic;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.math.BigDecimal;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
@@ -45,6 +86,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
+import java.util.stream.Stream;
import static
org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_COLUMN_STATS;
import static
org.apache.hudi.metadata.HoodieTableMetadataUtil.PARTITION_NAME_PARTITION_STATS;
@@ -52,8 +94,17 @@ import static
org.apache.hudi.metadata.HoodieTableMetadataUtil.getIndexVersionOp
import static
org.apache.hudi.metadata.SecondaryIndexKeyUtils.constructSecondaryIndexKey;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockConstruction;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class TestHoodieTableMetadataUtil {
@@ -432,4 +483,519 @@ class TestHoodieTableMetadataUtil {
TimeZone.setDefault(originalTimeZone);
}
}
+
+ @Test
+ void testColumnStatsValueValidation() {
+
assertFalse(HoodieTableMetadataUtil.getColumnStatsValueAsString(null).isPresent());
+ assertThrows(HoodieNotSupportedException.class,
+ () -> HoodieTableMetadataUtil.getColumnStatsValueAsString(new
Object()));
+ }
+
+ @Test
+ void testWritePartitionPathsIncludeNonPartitionedTableIdentifier() {
+ HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
+ commitMetadata.addWriteStat("", new HoodieWriteStat());
+ commitMetadata.addWriteStat("year=2026", new HoodieWriteStat());
+
+ assertEquals(
+ new java.util.HashSet<>(Arrays.asList("", "year=2026")),
+
HoodieTableMetadataUtil.getWritePartitionPaths(Collections.singletonList(commitMetadata)));
+ }
+
+ @Test
+ void testDecimalUpcastValidation() {
+ assertThrows(AvroTypeException.class,
+ () -> HoodieTableMetadataUtil.tryUpcastDecimal(
+ new BigDecimal("1.23"), LogicalTypes.decimal(5, 1)));
+ assertThrows(AvroTypeException.class,
+ () -> HoodieTableMetadataUtil.tryUpcastDecimal(
+ new BigDecimal("123"), LogicalTypes.decimal(3, 1)));
+ assertThrows(AvroTypeException.class,
+ () -> HoodieTableMetadataUtil.tryUpcastDecimal(
+ new BigDecimal("1234"), LogicalTypes.decimal(3, 0)));
+ }
+
+ @Test
+ void testComparableCoercion() {
+ assertNull(HoodieTableMetadataUtil.coerceToComparable(
+ HoodieSchema.create(HoodieSchemaType.INT), null));
+ assertEquals(1, HoodieTableMetadataUtil.coerceToComparable(
+ HoodieSchema.create(HoodieSchemaType.INT), true));
+ assertEquals(0L, HoodieTableMetadataUtil.coerceToComparable(
+ HoodieSchema.create(HoodieSchemaType.LONG), false));
+ assertEquals(1.5f, HoodieTableMetadataUtil.coerceToComparable(
+ HoodieSchema.create(HoodieSchemaType.FLOAT), 1.5d));
+ assertEquals(2.5d, HoodieTableMetadataUtil.coerceToComparable(
+ HoodieSchema.create(HoodieSchemaType.DOUBLE), 2.5f));
+ assertEquals(1.0f, HoodieTableMetadataUtil.coerceToComparable(
+ HoodieSchema.create(HoodieSchemaType.FLOAT), true));
+ assertEquals(0.0d, HoodieTableMetadataUtil.coerceToComparable(
+ HoodieSchema.create(HoodieSchemaType.DOUBLE), false));
+ assertNull(HoodieTableMetadataUtil.coerceToComparable(
+ HoodieSchema.create(HoodieSchemaType.NULL), "ignored"));
+ }
+
+ @Test
+ void testFileGroupCountBoundsAndInflightWriteStatusTracking() {
+ assertEquals(10, HoodieTableMetadataUtil.estimateFileGroupCount(
+ MetadataPartitionType.RECORD_INDEX, () -> 10_000L, 1, 1, 10, 1.0f,
100));
+ assertEquals(5, HoodieTableMetadataUtil.estimateFileGroupCount(
+ MetadataPartitionType.RECORD_INDEX, () -> 500L, 1, 2, 10, 1.0f, 100));
+
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(metaClient.getTableConfig()).thenReturn(tableConfig);
+
when(tableConfig.isMetadataPartitionAvailable(MetadataPartitionType.RECORD_INDEX)).thenReturn(false);
+ when(tableConfig.getMetadataPartitionsInflight())
+
.thenReturn(Collections.singleton(MetadataPartitionType.RECORD_INDEX.getPartitionPath()));
+
+
assertTrue(HoodieTableMetadataUtil.getMetadataPartitionsNeedingWriteStatusTracking(
+ HoodieMetadataConfig.newBuilder().enable(false).build(), metaClient));
+ }
+
+ @Test
+ @SuppressWarnings("deprecation")
+ void testGenerateKeyPrefixesMatchesRawKeyEncoding() {
+ List<String> columns = Arrays.asList("c1", "c2");
+ assertEquals(
+ HoodieTableMetadataUtil.generateColumnStatsKeys(columns,
"partition").stream()
+ .map(ColumnStatsIndexPrefixRawKey::encode)
+ .collect(java.util.stream.Collectors.toList()),
+ HoodieTableMetadataUtil.generateKeyPrefixes(columns, "partition"));
+ }
+
+ @Test
+ void testCollectColumnRangeMetadata() {
+ HoodieSchema recordSchema = mock(HoodieSchema.class);
+ StorageConfiguration<?> storageConfig = mock(StorageConfiguration.class);
+ when(storageConfig.getString(
+
org.apache.hudi.common.config.HoodieStorageConfig.WRITE_UTC_TIMEZONE.key(),
+
org.apache.hudi.common.config.HoodieStorageConfig.WRITE_UTC_TIMEZONE.defaultValue().toString()))
+ .thenReturn("UTC");
+
+ HoodieRecord<?> record = mock(HoodieRecord.class);
+ when(record.getRecordType()).thenReturn(HoodieRecordType.FLINK);
+ when(record.getColumnValueAsJava(
+ org.mockito.ArgumentMatchers.eq(recordSchema),
+ org.mockito.ArgumentMatchers.eq("id"),
+ org.mockito.ArgumentMatchers.any()))
+ .thenReturn(7);
+
+ HoodieSchemaField idField = HoodieSchemaField.of(
+ "id", HoodieSchema.create(HoodieSchemaType.INT), null, null);
+ HoodieSchemaField unsupportedField = HoodieSchemaField.of(
+ "attributes",
+ HoodieSchema.createMap(HoodieSchema.create(HoodieSchemaType.STRING)),
+ null,
+ null);
+ Map<String, HoodieColumnRangeMetadata<Comparable>> stats =
+ HoodieTableMetadataUtil.collectColumnRangeMetadata(
+ Collections.<HoodieRecord>singletonList(record).iterator(),
+ Arrays.asList(Pair.of("id", idField), Pair.of("attributes",
unsupportedField)),
+ "file.parquet",
+ recordSchema,
+ storageConfig,
+ HoodieIndexVersion.V1);
+
+ assertEquals(7, stats.get("id").getMinValue());
+ assertEquals(7, stats.get("id").getMaxValue());
+ assertNull(stats.get("attributes").getMinValue());
+ }
+
+ @Test
+ void testFilePartitionRecordConversionHandlesDeletesAndAppends() {
+ Map<String, List<String>> deletedFiles = new HashMap<>();
+ deletedFiles.put("p1", Collections.singletonList("old.parquet"));
+ Map<String, List<FileInfoAndPartition>> appendedFiles = new HashMap<>();
+ appendedFiles.put("p1", Collections.singletonList(
+ FileInfoAndPartition.of("p1", "new.parquet", 10L)));
+ appendedFiles.put("p2", Collections.singletonList(
+ FileInfoAndPartition.of("p2", "other.parquet", 20L)));
+
+ List<HoodieRecord> records =
HoodieTableMetadataUtil.convertFilesToFilesPartitionRecords(
+ deletedFiles, appendedFiles, "001", "test");
+
+ assertEquals(2, records.size());
+ assertFalse(appendedFiles.containsKey("p1"));
+ }
+
+ @Test
+ void testBloomAndColumnStatsConversionFast() {
+ HoodieLocalEngineContext engineContext =
+ new HoodieLocalEngineContext(mock(StorageConfiguration.class));
+ Map<String, List<String>> deletedFiles = new HashMap<>();
+ deletedFiles.put("p1", Arrays.asList("file.log.1",
"file_1-0-1_001.parquet"));
+
+ HoodieData<HoodieRecord> records =
HoodieTableMetadataUtil.convertFilesToBloomFilterRecords(
+ engineContext,
+ deletedFiles,
+ Collections.emptyMap(),
+ "001",
+ mock(HoodieTableMetaClient.class),
+ 2,
+ "SIMPLE");
+ assertEquals(1, records.collectAsList().size());
+
+ assertTrue(HoodieTableMetadataUtil.convertFilesToColumnStatsRecords(
+ engineContext,
+ Collections.emptyMap(),
+ Collections.emptyMap(),
+ mock(HoodieTableMetaClient.class),
+ 1,
+ 1024,
+ Collections.singletonList("id")).collectAsList().isEmpty());
+ }
+
+ @Test
+ void testFilesPartitionAvailability() {
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ when(metaClient.getTableConfig()).thenReturn(tableConfig);
+ when(tableConfig.getMetadataPartitions())
+
.thenReturn(Collections.singleton(HoodieTableMetadataUtil.PARTITION_NAME_FILES));
+
+ assertTrue(HoodieTableMetadataUtil.isFilesPartitionAvailable(metaClient));
+ }
+
+ @Test
+ void testMetadataTableDeletionOutcomes() throws Exception {
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ HoodieStorage storage = mock(HoodieStorage.class);
+ when(metaClient.getBasePath()).thenReturn(new StoragePath("/table"));
+ when(metaClient.getTableConfig()).thenReturn(tableConfig);
+ when(metaClient.getStorage()).thenReturn(storage);
+
+ when(storage.exists(any(StoragePath.class))).thenReturn(false);
+ assertNull(HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null,
false));
+
+ reset(storage);
+ when(storage.exists(any(StoragePath.class))).thenThrow(new
FileNotFoundException("missing"));
+ assertNull(HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null,
false));
+
+ reset(storage);
+ when(storage.exists(any(StoragePath.class))).thenThrow(new
IOException("check failed"));
+ assertThrows(HoodieMetadataException.class,
+ () -> HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null,
false));
+
+ reset(storage);
+ when(storage.exists(any(StoragePath.class))).thenReturn(true);
+ when(storage.rename(any(StoragePath.class),
any(StoragePath.class))).thenReturn(true);
+ assertTrue(HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null,
true)
+ .contains(".metadata_"));
+
+ reset(storage);
+ when(storage.exists(any(StoragePath.class))).thenReturn(true);
+ when(storage.rename(any(StoragePath.class), any(StoragePath.class)))
+ .thenThrow(new IOException("rename failed"));
+ assertNull(HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null,
true));
+
+ reset(storage);
+ when(storage.exists(any(StoragePath.class))).thenReturn(true);
+ org.mockito.Mockito.doThrow(new IOException("delete failed"))
+ .when(storage).deleteDirectory(any(StoragePath.class));
+ assertThrows(HoodieMetadataException.class,
+ () -> HoodieTableMetadataUtil.deleteMetadataTable(metaClient, null,
false));
+ }
+
+ @Test
+ void testMetadataPartitionDeletionOutcomes() throws Exception {
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ HoodieTableConfig tableConfig = mock(HoodieTableConfig.class);
+ HoodieStorage storage = mock(HoodieStorage.class);
+ when(metaClient.getBasePath()).thenReturn(new StoragePath("/table"));
+ when(metaClient.getTableConfig()).thenReturn(tableConfig);
+ when(metaClient.getStorage()).thenReturn(storage);
+ String partition = MetadataPartitionType.COLUMN_STATS.getPartitionPath();
+
+ when(storage.exists(any(StoragePath.class))).thenReturn(false);
+ assertNull(HoodieTableMetadataUtil.deleteMetadataTablePartition(
+ metaClient, null, MetadataPartitionType.FILES.getPartitionPath(),
false));
+
+ reset(storage);
+ when(storage.exists(any(StoragePath.class))).thenThrow(new
FileNotFoundException("missing"));
+ assertNull(HoodieTableMetadataUtil.deleteMetadataTablePartition(
+ metaClient, null, partition, false));
+
+ reset(storage);
+ when(storage.exists(any(StoragePath.class))).thenThrow(new
IOException("check failed"));
+ assertThrows(HoodieMetadataException.class,
+ () -> HoodieTableMetadataUtil.deleteMetadataTablePartition(
+ metaClient, null, partition, false));
+
+ reset(storage);
+ when(storage.exists(any(StoragePath.class))).thenReturn(true);
+ when(storage.rename(any(StoragePath.class),
any(StoragePath.class))).thenReturn(true);
+ assertTrue(HoodieTableMetadataUtil.deleteMetadataTablePartition(
+ metaClient, null, partition, true).contains(".metadata_"));
+
+ reset(storage);
+ when(storage.exists(any(StoragePath.class))).thenReturn(true);
+ when(storage.rename(any(StoragePath.class), any(StoragePath.class)))
+ .thenThrow(new IOException("rename failed"));
+ assertNull(HoodieTableMetadataUtil.deleteMetadataTablePartition(
+ metaClient, null, partition, true));
+
+ reset(storage);
+ when(storage.exists(any(StoragePath.class))).thenReturn(true);
+ org.mockito.Mockito.doThrow(new IOException("delete failed"))
+ .when(storage).deleteDirectory(any(StoragePath.class));
+ assertThrows(HoodieMetadataException.class,
+ () -> HoodieTableMetadataUtil.deleteMetadataTablePartition(
+ metaClient, null, partition, false));
+ }
+
+ @Test
+ void testRecordKeyReadSchemaFailureIsWrapped() {
+ HoodieEngineContext engineContext = mock(HoodieEngineContext.class);
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ when(metaClient.getBasePath()).thenReturn(new StoragePath("/table"));
+
when(metaClient.getStorageConf()).thenReturn(mock(StorageConfiguration.class));
+
+ assertThrows(org.apache.hudi.exception.HoodieException.class,
+ () -> HoodieTableMetadataUtil.readRecordKeysFromFileSlices(
+ engineContext,
+ Collections.singletonList(Pair.of("partition",
mock(FileSlice.class))),
+ 1,
+ "test",
+ metaClient,
+ false));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ void testPartitionStatsConversionFailureIsWrapped() {
+ HoodiePairData<String, List<HoodieColumnRangeMetadata<Comparable>>>
pairData =
+ mock(HoodiePairData.class);
+ when(pairData.flatMapValues(any())).thenThrow(new
RuntimeException("conversion failed"));
+
+ assertThrows(org.apache.hudi.exception.HoodieException.class,
+ () -> HoodieTableMetadataUtil.convertMetadataToPartitionStatsRecords(
+ pairData,
+ mock(HoodieTableMetaClient.class),
+ Collections.emptyMap(),
+ HoodieIndexVersion.V1));
+ }
+
+ @Test
+ void testMergeColumnStatsTombstoneWins() {
+ HoodieMetadataColumnStats previous = HoodieMetadataColumnStats.newBuilder()
+ .setColumnName("column")
+ .setIsDeleted(false)
+ .build();
+ HoodieMetadataColumnStats tombstone =
HoodieMetadataColumnStats.newBuilder()
+ .setColumnName("column")
+ .setIsDeleted(true)
+ .build();
+
+ assertEquals(tombstone,
HoodieTableMetadataUtil.mergeColumnStatsRecords(previous, tombstone));
+ assertEquals(previous,
HoodieTableMetadataUtil.mergeColumnStatsRecords(tombstone, previous));
+ }
+
+ @Test
+ void testFileSliceAndSchemaResolutionEdge() {
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ HoodieTimeline timeline = mock(HoodieTimeline.class);
+ when(metaClient.getCommitsTimeline()).thenReturn(timeline);
+ when(timeline.filterCompletedInstants()).thenReturn(timeline);
+ when(timeline.countInstants()).thenReturn(1);
+ when(metaClient.getBasePath()).thenReturn(new StoragePath("/table"));
+ assertThrows(org.apache.hudi.exception.HoodieException.class,
+ () -> HoodieTableMetadataUtil.tryResolveSchemaForTable(metaClient));
+
+ HoodieTableFileSystemView fileSystemView =
mock(HoodieTableFileSystemView.class);
+ HoodieActiveTimeline activeTimeline = mock(HoodieActiveTimeline.class);
+ when(metaClient.getActiveTimeline()).thenReturn(activeTimeline);
+ when(activeTimeline.filterCompletedInstants()).thenReturn(activeTimeline);
+ when(activeTimeline.lastInstant()).thenReturn(Option.empty());
+ assertTrue(HoodieTableMetadataUtil.getPartitionLatestMergedFileSlices(
+ metaClient, fileSystemView, "files").isEmpty());
+ }
+
+ @Test
+ void testEmptyLogInputsAvoidReaderConstruction() {
+ Pair<java.util.Set<String>, java.util.Set<String>> changes =
+ HoodieTableMetadataUtil.getRevivedAndDeletedKeysFromMergedLogs(
+ mock(HoodieTableMetaClient.class),
+ "001",
+ Collections.singletonList("previous.log"),
+ Option.empty(),
+ Collections.singletonList("current.log"),
+ "partition",
+ mock(org.apache.hudi.common.engine.HoodieReaderContext.class));
+
+ assertTrue(changes.getLeft().isEmpty());
+ assertTrue(changes.getRight().isEmpty());
+ }
+
+ @Test
+ void testRollbackPlanFallbackAndReadFailure() throws Exception {
+ Method method = HoodieTableMetadataUtil.class.getDeclaredMethod(
+ "getRollbackedCommits",
+ HoodieInstant.class,
+ HoodieActiveTimeline.class,
+ InstantGenerator.class);
+ method.setAccessible(true);
+
+ HoodieInstant completed = mock(HoodieInstant.class);
+ HoodieInstant requested = mock(HoodieInstant.class);
+ HoodieActiveTimeline timeline = mock(HoodieActiveTimeline.class);
+ InstantGenerator instantGenerator = mock(InstantGenerator.class);
+ HoodieRollbackPlan rollbackPlan = mock(HoodieRollbackPlan.class);
+ HoodieInstantInfo instantInfo = mock(HoodieInstantInfo.class);
+ when(completed.getAction()).thenReturn(HoodieTimeline.ROLLBACK_ACTION);
+ when(completed.requestedTime()).thenReturn("002");
+ when(timeline.readRollbackMetadata(completed)).thenThrow(new
IOException("empty rollback"));
+ when(instantGenerator.createNewInstant(
+ HoodieInstant.State.REQUESTED, HoodieTimeline.ROLLBACK_ACTION, "002"))
+ .thenReturn(requested);
+ when(timeline.readRollbackPlan(requested)).thenReturn(rollbackPlan);
+ when(rollbackPlan.getInstantToRollback()).thenReturn(instantInfo);
+ when(instantInfo.getCommitTime()).thenReturn("001");
+ assertEquals(Collections.singletonList("001"), method.invoke(
+ null, completed, timeline, instantGenerator));
+
+ when(completed.getAction()).thenReturn(HoodieTimeline.RESTORE_ACTION);
+ when(timeline.readRestoreMetadata(completed)).thenThrow(new
IOException("broken restore"));
+ InvocationTargetException exception = assertThrows(
+ InvocationTargetException.class,
+ () -> method.invoke(null, completed, timeline, instantGenerator));
+ assertTrue(exception.getCause() instanceof HoodieMetadataException);
+ }
+
+ @Test
+ void testMetadataPartitionExistenceFailureIsWrapped() throws Exception {
+ HoodieEngineContext context = mock(HoodieEngineContext.class);
+ StorageConfiguration<?> storageConfiguration =
mock(StorageConfiguration.class);
+ HoodieStorage storage = mock(HoodieStorage.class);
+
org.mockito.Mockito.doReturn(storageConfiguration).when(context).getStorageConf();
+ when(storage.exists(any(StoragePath.class))).thenThrow(new
IOException("failed"));
+
+ try (MockedStatic<HoodieStorageUtils> storageUtils =
mockStatic(HoodieStorageUtils.class)) {
+ storageUtils.when(() -> HoodieStorageUtils.getStorage(any(String.class),
any()))
+ .thenReturn(storage);
+ assertThrows(org.apache.hudi.exception.HoodieIOException.class,
+ () -> HoodieTableMetadataUtil.metadataPartitionExists(
+ "/table", context,
MetadataPartitionType.FILES.getPartitionPath()));
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ void testDeletedFileStatsCreateStubs() throws Exception {
+ Method method = HoodieTableMetadataUtil.class.getDeclaredMethod(
+ "getFileStatsRangeMetadata",
+ String.class,
+ String.class,
+ HoodieTableMetaClient.class,
+ List.class,
+ boolean.class,
+ int.class,
+ HoodieIndexVersion.class);
+ method.setAccessible(true);
+
+ List<HoodieColumnRangeMetadata<Comparable>> stats =
+ (List<HoodieColumnRangeMetadata<Comparable>>) method.invoke(
+ null,
+ "partition",
+ "file.parquet",
+ mock(HoodieTableMetaClient.class),
+ Arrays.asList("c1", "c2"),
+ true,
+ 1024,
+ HoodieIndexVersion.V1);
+ assertEquals(2, stats.size());
+ }
+
+ @Test
+ void testCommitPartitionExtraction() throws Exception {
+ HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
+ commitMetadata.addWriteStat("", new HoodieWriteStat());
+ commitMetadata.addWriteStat("partition", new HoodieWriteStat());
+ Method method = HoodieTableMetadataUtil.class.getDeclaredMethod(
+ "getPartitionsAdded", HoodieCommitMetadata.class);
+ method.setAccessible(true);
+
+ assertEquals(
+ new
java.util.HashSet<>(Arrays.asList(HoodieTableMetadata.NON_PARTITIONED_NAME,
"partition")),
+ new java.util.HashSet<>((List<String>) method.invoke(null,
commitMetadata)));
+ }
+
+ @Test
+ void testInflightFileSliceViewIsClosedWhenCreatedInternally() {
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ HoodieTableFileSystemView fileSystemView =
mock(HoodieTableFileSystemView.class);
+ when(fileSystemView.getLatestFileSlicesIncludingInflight("files"))
+ .thenReturn(Stream.empty());
+
+ try (MockedStatic<HoodieTableMetadataUtil> util =
+ mockStatic(HoodieTableMetadataUtil.class,
org.mockito.Answers.CALLS_REAL_METHODS)) {
+ util.when(() ->
HoodieTableMetadataUtil.getFileSystemViewForMetadataTable(metaClient))
+ .thenReturn(fileSystemView);
+
assertTrue(HoodieTableMetadataUtil.getPartitionLatestFileSlicesIncludingInflight(
+ metaClient, Option.empty(), "files").isEmpty());
+ }
+
+ verify(fileSystemView).close();
+ }
+
+ @Test
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ void testLogOnlyFileSliceRecordKeys() throws Exception {
+ HoodieEngineContext engineContext = mock(HoodieEngineContext.class);
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ HoodieSchema schema = mock(HoodieSchema.class);
+ FileSlice fileSlice = mock(FileSlice.class);
+ List<Pair<String, FileSlice>> slices =
+ Collections.singletonList(Pair.of("partition", fileSlice));
+ when(metaClient.getBasePath()).thenReturn(new StoragePath("/table"));
+ StorageConfiguration<?> storageConfiguration =
mock(StorageConfiguration.class);
+ when(storageConfiguration.getEnum(
+ any(String.class), any(ExternalSpillableMap.DiskMapType.class)))
+ .thenAnswer(invocation -> invocation.getArgument(1));
+
org.mockito.Mockito.doReturn(storageConfiguration).when(metaClient).getStorageConf();
+ HoodieActiveTimeline timeline = mock(HoodieActiveTimeline.class);
+ when(metaClient.getActiveTimeline()).thenReturn(timeline);
+ when(timeline.filterCompletedInstants()).thenReturn(timeline);
+ when(timeline.lastInstant()).thenReturn(Option.empty());
+ when(fileSlice.getBaseFile()).thenReturn(Option.empty());
+ when(fileSlice.getLogFiles()).thenReturn(Stream.empty());
+ when(fileSlice.getPartitionPath()).thenReturn("partition");
+ when(fileSlice.getFileId()).thenReturn("file-id");
+ when(fileSlice.getBaseInstantTime()).thenReturn("20240101000000000");
+ org.mockito.Mockito.doReturn(HoodieListData.eager(slices))
+ .when(engineContext).parallelize(anyList(), anyInt());
+
+ ReaderContextFactory readerContextFactory =
mock(ReaderContextFactory.class);
+ HoodieReaderContext readerContext = mock(HoodieReaderContext.class);
+ org.mockito.Mockito.doReturn(readerContextFactory)
+ .when(engineContext).getReaderContextFactory(metaClient);
+ when(readerContextFactory.getContext()).thenReturn(readerContext);
+
+ HoodieFileGroupReader.HoodieFileGroupReaderBuilder builder =
+ mock(HoodieFileGroupReader.HoodieFileGroupReaderBuilder.class);
+ HoodieFileGroupReader fileGroupReader = mock(HoodieFileGroupReader.class);
+ when(builder.withReaderContext(any())).thenReturn(builder);
+ when(builder.withHoodieTableMetaClient(any())).thenReturn(builder);
+ when(builder.withBaseFileOption(any())).thenReturn(builder);
+ when(builder.withLogFiles(any())).thenReturn(builder);
+ when(builder.withPartitionPath(any())).thenReturn(builder);
+ when(builder.withDataSchema(any())).thenReturn(builder);
+ when(builder.withRequestedSchema(any())).thenReturn(builder);
+ when(builder.withLatestCommitTime(any())).thenReturn(builder);
+ when(builder.withProps(any())).thenReturn(builder);
+ when(builder.build()).thenReturn(fileGroupReader);
+ when(fileGroupReader.getClosableKeyIterator())
+ .thenReturn(ClosableIterator.wrap(Collections.emptyIterator()));
+
+ try (MockedConstruction<TableSchemaResolver> ignored =
+ mockConstruction(TableSchemaResolver.class,
+ (resolver, context) ->
when(resolver.getTableSchema()).thenReturn(schema));
+ MockedStatic<HoodieFileGroupReader> readerStatic =
+ mockStatic(HoodieFileGroupReader.class)) {
+ readerStatic.when(HoodieFileGroupReader::builder).thenReturn(builder);
+ assertTrue(HoodieTableMetadataUtil.readRecordKeysFromFileSlices(
+ engineContext, slices, 1, "test", metaClient,
false).collectAsList().isEmpty());
+ }
+ }
}
diff --git
a/hudi-common/src/test/java/org/apache/hudi/metadata/TestMetadataPartitionType.java
b/hudi-common/src/test/java/org/apache/hudi/metadata/TestMetadataPartitionType.java
index 92ce2d2693b7..8aa5879794c8 100644
---
a/hudi-common/src/test/java/org/apache/hudi/metadata/TestMetadataPartitionType.java
+++
b/hudi-common/src/test/java/org/apache/hudi/metadata/TestMetadataPartitionType.java
@@ -19,6 +19,8 @@
package org.apache.hudi.metadata;
+import org.apache.hudi.avro.model.HoodieMetadataBloomFilter;
+import org.apache.hudi.avro.model.HoodieMetadataRecord;
import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.model.HoodieIndexDefinition;
import org.apache.hudi.common.model.HoodieIndexMetadata;
@@ -28,6 +30,8 @@ import org.apache.hudi.common.table.HoodieTableVersion;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
@@ -35,6 +39,7 @@ import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.Mockito;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -320,4 +325,76 @@ public class TestMetadataPartitionType {
}
}
}
+
+ @Test
+ public void testProjectedPayloadConstruction() {
+ HoodieMetadataPayload payload = new HoodieMetadataPayload(Option.empty());
+ Schema projectedSchema = Schema.createRecord("ProjectedMetadataRecord",
null, null, false);
+ projectedSchema.setFields(Collections.emptyList());
+ GenericData.Record projectedRecord = new
GenericData.Record(projectedSchema);
+
+ MetadataPartitionType.FILES.constructMetadataPayload(payload,
projectedRecord);
+ MetadataPartitionType.COLUMN_STATS.constructMetadataPayload(payload,
projectedRecord);
+ assertThrows(UnsupportedOperationException.class,
+ () ->
MetadataPartitionType.EXPRESSION_INDEX.constructMetadataPayload(payload,
projectedRecord));
+
+ GenericData.Record invalidBloomFilterRecord =
+ new GenericData.Record(HoodieMetadataRecord.getClassSchema());
+ assertThrows(IllegalArgumentException.class,
+ () ->
MetadataPartitionType.BLOOM_FILTERS.constructMetadataPayload(payload,
invalidBloomFilterRecord));
+ assertThrows(IllegalArgumentException.class, () ->
MetadataPartitionType.get(Integer.MAX_VALUE));
+ }
+
+ @Test
+ public void testBloomFilterCombinationAndAllPartitionsEnablement() {
+ HoodieMetadataPayload older = new HoodieMetadataPayload("key",
+ new HoodieMetadataBloomFilter("SIMPLE", "1", ByteBuffer.wrap(new
byte[] {1}), false));
+ HoodieMetadataPayload newer = new HoodieMetadataPayload("key",
+ new HoodieMetadataBloomFilter("SIMPLE", "2", ByteBuffer.wrap(new
byte[] {2}), false));
+
+ HoodieMetadataPayload combined =
MetadataPartitionType.BLOOM_FILTERS.combineMetadataPayloads(older, newer);
+
+ assertEquals(newer.getBloomFilterMetadata(),
combined.getBloomFilterMetadata());
+ assertTrue(MetadataPartitionType.ALL_PARTITIONS.isMetadataPartitionEnabled(
+ HoodieMetadataConfig.newBuilder().enable(true).build(),
mock(HoodieTableConfig.class)));
+ }
+
+ @Test
+ public void testNewIndexDefinitionChecks() {
+ HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class);
+ HoodieIndexDefinition secondaryIndex = createIndexDefinition(
+ MetadataPartitionType.SECONDARY_INDEX, "existing",
HoodieTableMetadataUtil.PARTITION_NAME_SECONDARY_INDEX,
+ null, Collections.singletonList("secondary_col"), null);
+ HoodieIndexDefinition expressionIndex = createIndexDefinition(
+ MetadataPartitionType.EXPRESSION_INDEX, "existing",
HoodieTableMetadataUtil.PARTITION_NAME_EXPRESSION_INDEX,
+ "lower", Collections.singletonList("expression_col"), null);
+ HoodieIndexMetadata indexMetadata = new
HoodieIndexMetadata(createIndexDefinitions(secondaryIndex, expressionIndex));
+ when(metaClient.getIndexMetadata()).thenReturn(Option.of(indexMetadata));
+
+ HoodieMetadataConfig secondaryConfig = HoodieMetadataConfig.newBuilder()
+ .withSecondaryIndexForColumn("secondary_col")
+ .build();
+
assertFalse(MetadataPartitionType.isNewSecondaryIndexDefinitionRequired(secondaryConfig,
metaClient));
+
+ HoodieMetadataConfig expressionConfig = HoodieMetadataConfig.newBuilder()
+ .withExpressionIndexColumn("expression_col")
+ .withExpressionIndexOptions(Collections.singletonMap("expr", "lower"))
+ .build();
+
assertFalse(MetadataPartitionType.isNewExpressionIndexDefinitionRequired(expressionConfig,
metaClient));
+
+ HoodieMetadataConfig differentExpressionConfig =
HoodieMetadataConfig.newBuilder()
+ .withExpressionIndexColumn("expression_col")
+ .withExpressionIndexOptions(Collections.singletonMap("expr", "upper"))
+ .build();
+
assertTrue(MetadataPartitionType.isNewExpressionIndexDefinitionRequired(differentExpressionConfig,
metaClient));
+
+ HoodieMetadataConfig noExpressionConfig = HoodieMetadataConfig.newBuilder()
+ .withExpressionIndexColumn("expression_col")
+ .build();
+
assertFalse(MetadataPartitionType.isNewExpressionIndexDefinitionRequired(noExpressionConfig,
metaClient));
+ }
+
+ private static Map<String, HoodieIndexDefinition>
createIndexDefinitions(HoodieIndexDefinition... definitions) {
+ return
Arrays.stream(definitions).collect(Collectors.toMap(HoodieIndexDefinition::getIndexName,
definition -> definition));
+ }
}
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestBaseTableMetadata.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestBaseTableMetadata.java
new file mode 100644
index 000000000000..f8115530e856
--- /dev/null
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestBaseTableMetadata.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metadata;
+
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.data.HoodieData;
+import org.apache.hudi.common.data.HoodieListData;
+import org.apache.hudi.common.data.HoodieListPairData;
+import org.apache.hudi.common.data.HoodiePairData;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.expression.Expression;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordGlobalLocation;
+import org.apache.hudi.common.schema.internal.Types;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.HoodieMetadataException;
+import org.apache.hudi.storage.HoodieStorage;
+import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class TestBaseTableMetadata {
+
+ @TempDir
+ Path tempDir;
+
+ private HoodieTableMetaClient metaClient;
+ private String basePath;
+
+ @BeforeEach
+ void setUp() throws Exception {
+ basePath = tempDir.toString();
+ metaClient = HoodieTestUtils.init(basePath);
+ }
+
+ @Test
+ void testReadFailuresAreWrappedWithMetadataContext() {
+ TestingTableMetadata metadata = newMetadata();
+ metadata.failSingleReads = true;
+ assertThrows(HoodieMetadataException.class,
metadata::getAllPartitionPaths);
+
+ metadata.failSingleReads = false;
+ metadata.failBulkReads = true;
+ assertThrows(HoodieMetadataException.class,
+ () ->
metadata.getAllFilesInPartitions(Collections.singletonList(basePath)));
+ }
+
+ @Test
+ void testDisabledIndexesAndEmptyBloomFilterLookup() {
+ TestingTableMetadata metadata = newMetadata();
+ assertFalse(metadata.getBloomFilter("", "file.parquet",
MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()).isPresent());
+ assertTrue(metadata.getBloomFilters(
+ Collections.singletonList(Pair.of("", "file.parquet")),
+ MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()).isEmpty());
+ assertTrue(metadata.getColumnStats(
+ Collections.singletonList(Pair.of("", "file.parquet")),
+ Collections.singletonList("column")).isEmpty());
+
+ metaClient.getTableConfig().setMetadataPartitionState(
+ metaClient, MetadataPartitionType.BLOOM_FILTERS.getPartitionPath(),
true);
+ metadata = newMetadata();
+ assertFalse(metadata.getBloomFilter(
+ "", "file.parquet",
MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()).isPresent());
+ assertTrue(metadata.getBloomFilters(
+ Collections.emptyList(),
MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()).isEmpty());
+ }
+
+ @Test
+ void testPayloadFailuresAndMissingColumnStats() {
+ HoodieMetadataPayload payload = new HoodieMetadataPayload(
+ "bad", MetadataPartitionType.FILES.getRecordType(),
Collections.emptyMap()) {
+ @Override
+ public List<StoragePathInfo> getFileList(
+ HoodieStorage storage, StoragePath partitionPath) {
+ throw new HoodieException("corrupt payload");
+ }
+ };
+
+ TestingTableMetadata firstMetadata = newMetadata();
+ firstMetadata.singleRecord = Option.of(payload);
+ assertThrows(HoodieException.class,
+ () -> firstMetadata.getAllFilesInPartition(new StoragePath(basePath)));
+
+ HoodieMetadataPayload inconsistentPayload =
+ HoodieMetadataPayload.createPartitionListRecord(
+ Collections.singletonList("ghost"), true).getData();
+ firstMetadata.singleRecord = Option.of(inconsistentPayload);
+ assertThrows(HoodieMetadataException.class,
firstMetadata::getAllPartitionPaths);
+
+ metaClient.getTableConfig().setMetadataPartitionState(
+ metaClient, MetadataPartitionType.COLUMN_STATS.getPartitionPath(),
true);
+ TestingTableMetadata metadata = newMetadata();
+ HoodieMetadataPayload missingColumnStats =
+ HoodieMetadataPayload.createPartitionFilesRecord(
+ "", Collections.singletonMap("file.parquet", 1L),
+ Collections.emptyList()).getData();
+ metadata.pairRecords = HoodieListPairData.eager(
+ Collections.singletonList(Pair.of("missing-key", missingColumnStats)));
+ assertTrue(metadata.getColumnStats(
+ Collections.singletonList(Pair.of("", "file.parquet")),
+ Collections.singletonList("column")).isEmpty());
+ }
+
+ @Test
+ void testProtectedReadContextAccessors() {
+ TestingTableMetadata metadata = newMetadata();
+ assertNotNull(metadata.storageConfiguration());
+ assertEquals("00000000000000", metadata.latestDataInstant());
+ }
+
+ @Test
+ void testHoodieBackedMetadataStaysDisabledWithoutMetadataTable() {
+ HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(
+ null,
+ metaClient.getStorage(),
+ HoodieMetadataConfig.newBuilder().enable(false).build(),
+ basePath);
+
+ assertFalse(metadata.isMetadataTableInitialized());
+ assertFalse(metadata.getSyncedInstantTime().isPresent());
+ assertFalse(metadata.getLatestCompactionTime().isPresent());
+ metadata.close();
+ }
+
+ private TestingTableMetadata newMetadata() {
+ return new TestingTableMetadata(
+ null, metaClient.getStorage(),
+ HoodieMetadataConfig.newBuilder()
+ .enable(true)
+ .ignoreSpuriousDeletes(false)
+ .build(),
+ basePath);
+ }
+
+ private static class TestingTableMetadata extends BaseTableMetadata {
+ private boolean failSingleReads;
+ private boolean failBulkReads;
+ private Option<HoodieMetadataPayload> singleRecord = Option.empty();
+ private HoodiePairData<String, HoodieMetadataPayload> pairRecords =
+ HoodieListPairData.eager(Collections.emptyList());
+
+ TestingTableMetadata(HoodieEngineContext engineContext,
+ HoodieStorage storage,
+ HoodieMetadataConfig metadataConfig,
+ String dataBasePath) {
+ super(engineContext, storage, metadataConfig, dataBasePath);
+ isMetadataTableInitialized = true;
+ }
+
+ @Override
+ protected Option<HoodieMetadataPayload> readFilesIndexRecords(String key,
String partitionName) {
+ if (failSingleReads) {
+ throw new HoodieException("single read failed");
+ }
+ return singleRecord;
+ }
+
+ @Override
+ public List<String> getPartitionPathWithPathPrefixUsingFilterExpression(
+ List<String> relativePathPrefixes,
+ Types.RecordType partitionFields,
+ Expression expression) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List<String> getPartitionPathWithPathPrefixes(List<String>
relativePathPrefixes) {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public HoodiePairData<String, HoodieMetadataPayload>
readIndexRecordsWithKeys(
+ HoodieData<? extends RawKey> rawKeys, String partitionName) {
+ if (failBulkReads) {
+ throw new HoodieException("bulk read failed");
+ }
+ return pairRecords;
+ }
+
+ @Override
+ protected HoodiePairData<String, HoodieMetadataPayload>
readIndexRecordsWithKeys(
+ HoodieData<? extends RawKey> rawKeys,
+ String partitionName,
+ Option<String> dataTablePartition) {
+ return readIndexRecordsWithKeys(rawKeys, partitionName);
+ }
+
+ @Override
+ public HoodiePairData<String, String>
readSecondaryIndexDataTableRecordKeysWithKeys(
+ HoodieData<String> keys, String partitionName) {
+ return HoodieListPairData.eager(Collections.emptyList());
+ }
+
+ @Override
+ public HoodiePairData<String, HoodieRecordGlobalLocation>
readSecondaryIndexLocationsWithKeys(
+ HoodieData<String> secondaryKeys, String partitionName) {
+ return HoodieListPairData.eager(Collections.emptyList());
+ }
+
+ @Override
+ public HoodiePairData<String, HoodieRecordGlobalLocation>
readRecordIndexLocationsWithKeys(
+ HoodieData<String> recordKeys) {
+ return HoodieListPairData.eager(Collections.emptyList());
+ }
+
+ @Override
+ public HoodiePairData<String, HoodieRecordGlobalLocation>
readRecordIndexLocationsWithKeys(
+ HoodieData<String> recordKeys, Option<String> dataTablePartition) {
+ return HoodieListPairData.eager(Collections.emptyList());
+ }
+
+ @Override
+ public HoodieData<HoodieRecord<HoodieMetadataPayload>>
getRecordsByKeyPrefixes(
+ HoodieData<? extends RawKey> rawKeys,
+ String partitionName,
+ boolean shouldLoadInMemory) {
+ return HoodieListData.eager(Collections.emptyList());
+ }
+
+ @Override
+ public Map<Pair<String, StoragePath>, List<StoragePathInfo>>
listPartitions(
+ List<Pair<String, StoragePath>> partitionPathList) {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public Option<String> getSyncedInstantTime() {
+ return Option.empty();
+ }
+
+ @Override
+ public Option<String> getLatestCompactionTime() {
+ return Option.empty();
+ }
+
+ @Override
+ public void reset() {
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public int getNumFileGroupsForPartition(MetadataPartitionType partition) {
+ return 0;
+ }
+
+ @Override
+ public Map<String, List<FileSlice>>
getBucketizedFileGroupsForPartitionedRLI(
+ MetadataPartitionType partition) {
+ return Collections.emptyMap();
+ }
+
+ StorageConfiguration<?> storageConfiguration() {
+ return getStorageConf();
+ }
+
+ String latestDataInstant() {
+ return getLatestDataInstantTime();
+ }
+ }
+}
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
index 48a2851c0924..30a26b65c9c9 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestFileSystemBackedTableMetadata.java
@@ -18,10 +18,13 @@
package org.apache.hudi.metadata;
+import org.apache.hudi.common.data.HoodieListData;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
@@ -256,4 +259,37 @@ public class TestFileSystemBackedTableMetadata extends
HoodieCommonTestHarness {
}
}
+ @Test
+ public void testMetadataIndexOperationsAreUnsupported() {
+ HoodieLocalEngineContext localEngineContext =
+ new HoodieLocalEngineContext(metaClient.getStorageConf());
+ FileSystemBackedTableMetadata metadata = new FileSystemBackedTableMetadata(
+ localEngineContext, metaClient.getTableConfig(),
metaClient.getStorage(), basePath);
+
+ Assertions.assertThrows(UnsupportedOperationException.class,
metadata::getSyncedInstantTime);
+ Assertions.assertThrows(UnsupportedOperationException.class,
metadata::getLatestCompactionTime);
+ Assertions.assertThrows(HoodieMetadataException.class,
+ () -> metadata.getBloomFilter("", "file.parquet",
MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()));
+ Assertions.assertThrows(HoodieMetadataException.class,
+ () -> metadata.getBloomFilters(Collections.emptyList(),
MetadataPartitionType.BLOOM_FILTERS.getPartitionPath()));
+ Assertions.assertThrows(HoodieMetadataException.class,
+ () -> metadata.getColumnStats(Collections.emptyList(), "column"));
+ Assertions.assertThrows(HoodieMetadataException.class,
+ () -> metadata.getColumnStats(Collections.emptyList(),
Collections.singletonList("column")));
+ Assertions.assertThrows(HoodieMetadataException.class,
+ () -> metadata.getRecordsByKeyPrefixes(
+ HoodieListData.eager(Collections.emptyList()),
MetadataPartitionType.FILES.getPartitionPath(), false));
+ Assertions.assertThrows(HoodieMetadataException.class,
+ () ->
metadata.readRecordIndexLocationsWithKeys(HoodieListData.eager(Collections.emptyList())));
+ Assertions.assertThrows(HoodieMetadataException.class,
+ () ->
metadata.readRecordIndexLocationsWithKeys(HoodieListData.eager(Collections.emptyList()),
Option.empty()));
+ Assertions.assertThrows(HoodieMetadataException.class,
+ () -> metadata.readSecondaryIndexLocationsWithKeys(
+ HoodieListData.eager(Collections.emptyList()),
MetadataPartitionType.SECONDARY_INDEX.getPartitionPath()));
+ Assertions.assertThrows(HoodieMetadataException.class,
+ () ->
metadata.getNumFileGroupsForPartition(MetadataPartitionType.FILES));
+ Assertions.assertThrows(HoodieMetadataException.class,
+ () ->
metadata.getBucketizedFileGroupsForPartitionedRLI(MetadataPartitionType.RECORD_INDEX));
+ }
+
}
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
index 21564b10b6df..45048d4e80b0 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieMetadataPayload.java
@@ -18,17 +18,24 @@
package org.apache.hudi.metadata;
+import org.apache.hudi.avro.model.HoodieMetadataRecord;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.schema.HoodieSchema;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.metadata.stats.HoodieColumnRangeMetadata;
import org.apache.hudi.metadata.stats.ValueMetadata;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.junit.jupiter.api.Test;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -325,6 +332,62 @@ public class TestHoodieMetadataPayload extends
HoodieCommonTestHarness {
assertEquals(newSecondaryIndexRecord.getData(),
combinedSecondaryIndexRecord.get().getData());
}
+ @Test
+ public void testPayloadAccessorsAndObjectMethods() {
+ HoodieMetadataPayload emptyPayload = new
HoodieMetadataPayload(Option.empty());
+ assertFalse(emptyPayload.getBloomFilterMetadata().isPresent());
+ assertFalse(emptyPayload.getColumnStatMetadata().isPresent());
+ assertFalse(emptyPayload.equals("not-a-payload"));
+ emptyPayload.hashCode();
+
+ HoodieMetadataPayload secondaryIndexPayload =
HoodieMetadataPayload.createSecondaryIndexRecord(
+ "record-key", "secondary-key",
MetadataPartitionType.SECONDARY_INDEX.getPartitionPath() + "test",
true).getData();
+ assertTrue(secondaryIndexPayload.isSecondaryIndexDeleted());
+ }
+
+ @Test
+ public void testInvalidRecordIndexInputs() {
+ assertThrows(HoodieMetadataException.class,
+ () ->
HoodieMetadataPayload.parseRecordIndexInstantTime("not-an-instant"));
+ assertThrows(HoodieMetadataException.class,
+ () -> HoodieMetadataPayload.createRecordIndexUpdate(
+ "record-key", PARTITION_NAME, "not-a-uuid", "20240101000000000",
0));
+ }
+
+ @Test
+ public void testProjectedInsertValueIncludesBloomFilter() throws IOException
{
+ HoodieMetadataPayload bloomFilterPayload =
HoodieMetadataPayload.createBloomFilterMetadataRecord(
+ PARTITION_NAME, "file-id_1-0-1_20240101000000000.parquet",
"20240101000000000", "SIMPLE",
+ ByteBuffer.wrap("bloom-data".getBytes()), false).getData();
+ Schema projectedSchema = HoodieSchemaUtils.addMetadataFields(
+
HoodieSchema.fromAvroSchema(HoodieMetadataRecord.getClassSchema())).toAvroSchema();
+
+ IndexedRecord projectedRecord =
bloomFilterPayload.getInsertValue(projectedSchema).get();
+
+ assertEquals(bloomFilterPayload.getBloomFilterMetadata().get(),
+ ((GenericRecord) projectedRecord).get("BloomFilterMetadata"));
+ }
+
+ @Test
+ public void testPayloadToStringForIndexedRecordTypes() {
+ HoodieMetadataPayload filesPayload =
HoodieMetadataPayload.createPartitionFilesRecord(
+ PARTITION_NAME, Collections.singletonMap("file.parquet", 10L),
Collections.singletonList("old.parquet")).getData();
+ assertTrue(filesPayload.toString().contains("creations=[file.parquet]"));
+ assertTrue(filesPayload.toString().contains("deletions=[old.parquet]"));
+
+ HoodieMetadataPayload bloomFilterPayload =
HoodieMetadataPayload.createBloomFilterMetadataRecord(
+ PARTITION_NAME, "file-id_1-0-1_20240101000000000.parquet",
"20240101000000000", "SIMPLE",
+ ByteBuffer.wrap("bloom-data".getBytes()), false).getData();
+ assertTrue(bloomFilterPayload.toString().contains("BloomFilter"));
+
+ HoodieColumnRangeMetadata<Comparable> columnRange =
HoodieColumnRangeMetadata.<Comparable>create(
+ "file.parquet", "column", 1, 2, 0, 2, 10, 10,
ValueMetadata.V1EmptyMetadata.get());
+ HoodieMetadataPayload columnStatsPayload =
+ (HoodieMetadataPayload) HoodieMetadataPayload.createColumnStatsRecords(
+ PARTITION_NAME, Collections.singletonList(columnRange),
false).findFirst().get().getData();
+ assertTrue(columnStatsPayload.toString().contains("ColStats"));
+ }
+
@Test
public void testConstructSecondaryIndexKey() {
// Simple case
