This is an automated email from the ASF dual-hosted git repository.
nsivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 5178a3bedae8 fix(metadata-table): add config to skip zero-size data
files on MDT initialization (#18611)
5178a3bedae8 is described below
commit 5178a3bedae848e1322842b1db5f1b03ee439b83
Author: Nada <[email protected]>
AuthorDate: Sun Jul 19 14:11:08 2026 -0400
fix(metadata-table): add config to skip zero-size data files on MDT
initialization (#18611)
Adds an opt-in config `hoodie.metadata.skip.zero.size.files.on.initialize`
(default false, advanced) that skips zero-size data files while listing
the
data table during MDT initialization. Prevents init failures on tables
carrying leftover zero-byte files, and emits
`skipped_zero_size_files_on_initialize`
with the skipped count.
Skip is scoped to the initialize path only — the restore-sync caller of
`listAllPartitionsFromFilesystem` passes `skipZeroSizeFiles=false`
unconditionally, so files already tracked in MDT are not spuriously
deleted.
Existing `DirectoryInfo` constructors delegate with `false` to preserve
current behavior.
---
.../metadata/HoodieBackedTableMetadataWriter.java | 16 +++++++++----
.../functional/TestHoodieMetadataBootstrap.java | 28 ++++++++++++++++++++++
.../hudi/common/config/HoodieMetadataConfig.java | 17 +++++++++++++
.../apache/hudi/metadata/model/DirectoryInfo.java | 17 +++++++++++--
4 files changed, 72 insertions(+), 6 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 3e76d2d2ef1d..d16f894d3ce5 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -404,7 +404,8 @@ public abstract class HoodieBackedTableMetadataWriter<I, O>
implements HoodieTab
} else {
// if auto initialization is enabled, then we need to list all
partitions from the file system
if (dataWriteConfig.getMetadataConfig().shouldAutoInitialize()) {
- partitionInfoList =
listAllPartitionsFromFilesystem(dataTableInstantTime, pendingDataInstants);
+ partitionInfoList =
listAllPartitionsFromFilesystem(dataTableInstantTime, pendingDataInstants,
+
dataWriteConfig.getMetadataConfig().shouldSkipZeroSizeFilesOnInitialize());
} else {
// if auto initialization is disabled, we can return an empty list
partitionInfoList = Collections.emptyList();
@@ -581,9 +582,12 @@ public abstract class HoodieBackedTableMetadataWriter<I,
O> implements HoodieTab
*
* @param initializationTime Files which have a timestamp after this are
neglected
* @param pendingDataInstants Pending instants on data set
+ * @param skipZeroSizeFiles Whether zero-size data files should be skipped
during listing. This should only be
+ * enabled on the initialize path; other callers
(e.g. restore) must pass false so that
+ * files already tracked in the metadata table
are not spuriously deleted.
* @return List consisting of {@code DirectoryInfo} for each partition found.
*/
- private List<DirectoryInfo> listAllPartitionsFromFilesystem(String
initializationTime, Set<String> pendingDataInstants) {
+ private List<DirectoryInfo> listAllPartitionsFromFilesystem(String
initializationTime, Set<String> pendingDataInstants, boolean skipZeroSizeFiles)
{
if (dataMetaClient.getActiveTimeline().countInstants() == 0) {
return Collections.emptyList();
}
@@ -595,6 +599,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O>
implements HoodieTab
StorageConfiguration<?> storageConf = dataMetaClient.getStorageConf();
final String dirFilterRegex =
dataWriteConfig.getMetadataConfig().getDirectoryFilterRegex();
StoragePath storageBasePath = dataMetaClient.getBasePath();
+ long totalZeroSizeFiles = 0;
while (!pathsToList.isEmpty()) {
// In each round we will list a section of directories
@@ -608,12 +613,13 @@ public abstract class HoodieBackedTableMetadataWriter<I,
O> implements HoodieTab
List<DirectoryInfo> processedDirectories =
engineContext.map(pathsToProcess, path -> {
HoodieStorage storage = HoodieStorageUtils.getStorage(path,
storageConf);
String relativeDirPath =
FSUtils.getRelativePartitionPath(storageBasePath, path);
- return new DirectoryInfo(relativeDirPath,
storage.listDirectEntries(path), initializationTime, pendingDataInstants);
+ return new DirectoryInfo(relativeDirPath,
storage.listDirectEntries(path), initializationTime, pendingDataInstants, true,
skipZeroSizeFiles);
}, numDirsToList);
// If the listing reveals a directory, add it to queue. If the listing
reveals a hoodie partition, add it to
// the results.
for (DirectoryInfo dirInfo : processedDirectories) {
+ totalZeroSizeFiles += dirInfo.getZeroSizeFileCount();
if (!dirFilterRegex.isEmpty()) {
final String relativePath = dirInfo.getRelativePath();
if (!relativePath.isEmpty() && relativePath.matches(dirFilterRegex))
{
@@ -632,6 +638,8 @@ public abstract class HoodieBackedTableMetadataWriter<I, O>
implements HoodieTab
}
}
+ final long zeroSizeCount = totalZeroSizeFiles;
+ metrics.ifPresent(m ->
m.incrementMetric("skipped_zero_size_files_on_initialize", zeroSizeCount));
return partitionsToBootstrap;
}
@@ -1185,7 +1193,7 @@ public abstract class HoodieBackedTableMetadataWriter<I,
O> implements HoodieTab
// Restore requires the existing pipelines to be shutdown. So we can
safely scan the dataset to find the current
// list of files in the filesystem.
- List<DirectoryInfo> dirInfoList =
listAllPartitionsFromFilesystem(instantTime, Collections.emptySet());
+ List<DirectoryInfo> dirInfoList =
listAllPartitionsFromFilesystem(instantTime, Collections.emptySet(), false);
Map<String, DirectoryInfo> dirInfoMap =
dirInfoList.stream().collect(Collectors.toMap(DirectoryInfo::getRelativePath,
Function.identity()));
dirInfoList.clear();
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java
index 28b325889072..742e14874377 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java
@@ -108,6 +108,34 @@ public class TestHoodieMetadataBootstrap extends
TestHoodieMetadataBase {
validateMetadata(testTable);
}
+ @Test
+ public void testMetadataSkipsZeroSizeFilesOnInitialize() throws Exception {
+ HoodieTableType tableType = COPY_ON_WRITE;
+ init(tableType, false);
+ doPreBootstrapWriteOperation(testTable, INSERT, "0000001");
+ doPreBootstrapWriteOperation(testTable, "0000002");
+ // Add a zero-size base file — bootstrap should skip it without failing.
+ String fileName = UUID.randomUUID().toString();
+ Path zeroSizeFilePath = FileCreateUtilsLegacy.getBaseFilePath(basePath,
"p1", "0000003", fileName);
+ FileCreateUtilsLegacy.createBaseFile(basePath, "p1", "0000003", fileName,
0);
+
+ writeConfig = getWriteConfigBuilder(true, true, false)
+ .withMetadataConfig(HoodieMetadataConfig.newBuilder()
+ .enable(true)
+ .withSkipZeroSizeFilesOnInitialize(true)
+ .build())
+ .build();
+ initWriteConfigAndMetatableWriter(writeConfig, true);
+ syncTableMetadata(writeConfig);
+
+ // Delete the zero-size file before validation — it was skipped in MDT and
must not
+ // exist on disk for the filesystem-vs-MDT consistency check to pass.
+ Files.delete(zeroSizeFilePath);
+ validateMetadata(testTable);
+ doWriteInsertAndUpsert(testTable);
+ validateMetadata(testTable);
+ }
+
@ParameterizedTest
@EnumSource(HoodieTableType.class)
public void testMetadataBootstrapInsertUpsertRollback(HoodieTableType
tableType) throws Exception {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
index ed69f556b03e..5eaca878e5fa 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
@@ -195,6 +195,14 @@ public final class HoodieMetadataConfig extends
HoodieConfig {
.sinceVersion("0.7.0")
.withDocumentation("Directories matching this regex, will be filtered
out when initializing metadata table from lake storage for the first time.");
+ public static final ConfigProperty<Boolean>
SKIP_ZERO_SIZE_FILES_ON_INITIALIZE = ConfigProperty
+ .key(METADATA_PREFIX + ".skip.zero.size.files.on.initialize")
+ .defaultValue(false)
+ .markAdvanced()
+ .sinceVersion("1.2.0")
+ .withDocumentation("When enabled, zero-size data files encountered while
listing the data table during "
+ + "metadata table initialization are skipped instead of being
recorded in the metadata table.");
+
public static final ConfigProperty<Integer> FILE_LISTING_PARALLELISM_VALUE =
ConfigProperty
.key("hoodie.file.listing.parallelism")
.defaultValue(200)
@@ -812,6 +820,10 @@ public final class HoodieMetadataConfig extends
HoodieConfig {
return getString(DIR_FILTER_REGEX);
}
+ public boolean shouldSkipZeroSizeFilesOnInitialize() {
+ return getBoolean(SKIP_ZERO_SIZE_FILES_ON_INITIALIZE);
+ }
+
public boolean shouldIgnoreSpuriousDeletes() {
return getBoolean(IGNORE_SPURIOUS_DELETES);
}
@@ -1194,6 +1206,11 @@ public final class HoodieMetadataConfig extends
HoodieConfig {
return this;
}
+ public Builder withSkipZeroSizeFilesOnInitialize(boolean
skipZeroSizeFiles) {
+ metadataConfig.setValue(SKIP_ZERO_SIZE_FILES_ON_INITIALIZE,
String.valueOf(skipZeroSizeFiles));
+ return this;
+ }
+
public Builder ignoreSpuriousDeletes(boolean
validateMetadataPayloadConsistency) {
metadataConfig.setValue(IGNORE_SPURIOUS_DELETES,
String.valueOf(validateMetadataPayloadConsistency));
return this;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/model/DirectoryInfo.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/model/DirectoryInfo.java
index 16d224d81208..3109adebe2b9 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metadata/model/DirectoryInfo.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metadata/model/DirectoryInfo.java
@@ -28,6 +28,7 @@ import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
import java.io.Serializable;
import java.util.ArrayList;
@@ -48,6 +49,7 @@ import static
org.apache.hudi.common.table.timeline.InstantComparison.compareTim
* a very large number of files are present in the dataset being initialized.
*/
@Getter
+@Slf4j
public class DirectoryInfo implements Serializable {
private static final long serialVersionUID = 1L;
@@ -59,9 +61,10 @@ public class DirectoryInfo implements Serializable {
private final List<StoragePath> subDirectories = new ArrayList<>();
// Is this a hoodie partition
private boolean isHoodiePartition = false;
+ private int zeroSizeFileCount = 0;
public DirectoryInfo(String relativePath, List<StoragePathInfo> pathInfos,
String maxInstantTime, Set<String> pendingDataInstants) {
- this(relativePath, pathInfos, maxInstantTime, pendingDataInstants, true);
+ this(relativePath, pathInfos, maxInstantTime, pendingDataInstants, true,
false);
}
/**
@@ -69,6 +72,11 @@ public class DirectoryInfo implements Serializable {
*/
public DirectoryInfo(String relativePath, List<StoragePathInfo> pathInfos,
String maxInstantTime, Set<String> pendingDataInstants,
boolean validateHoodiePartitions) {
+ this(relativePath, pathInfos, maxInstantTime, pendingDataInstants,
validateHoodiePartitions, false);
+ }
+
+ public DirectoryInfo(String relativePath, List<StoragePathInfo> pathInfos,
String maxInstantTime, Set<String> pendingDataInstants,
+ boolean validateHoodiePartitions, boolean
skipZeroSizeFiles) {
this.relativePath = relativePath;
// Pre-allocate with the maximum length possible
@@ -89,7 +97,12 @@ public class DirectoryInfo implements Serializable {
String dataFileCommitTime =
FSUtils.getCommitTime(pathInfo.getPath().getName());
// Limit the file listings to files which were created by successful
commits before the maxInstant time.
if (!pendingDataInstants.contains(dataFileCommitTime) &&
compareTimestamps(dataFileCommitTime, LESSER_THAN_OR_EQUALS, maxInstantTime)) {
- filenameToSizeMap.put(pathInfo.getPath().getName(),
pathInfo.getLength());
+ if (pathInfo.getLength() > 0 || !skipZeroSizeFiles) {
+ filenameToSizeMap.put(pathInfo.getPath().getName(),
pathInfo.getLength());
+ } else {
+ log.warn("Skipping zero-size data file during MDT bootstrap: {}",
pathInfo.getPath());
+ zeroSizeFileCount++;
+ }
}
}
}
