This is an automated email from the ASF dual-hosted git repository.
vinish pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
The following commit(s) were added to refs/heads/main by this push:
new 2ae96426 Handle empty EarliestCommitToRetain (#732)
2ae96426 is described below
commit 2ae964266d587e33d7e1b6bf8b22356afaac7341
Author: Alex R <[email protected]>
AuthorDate: Wed Apr 8 09:57:03 2026 -0700
Handle empty EarliestCommitToRetain (#732)
* handle empty EarliestCommitToRetain
* Fix build failures: qualify GREATER_THAN and fix test mocks
* Use PathBasedPartitionSpecExtractor to align with main branch rename
---------
Co-authored-by: Vinish Reddy <[email protected]>
---
.../apache/xtable/hudi/HudiConversionSource.java | 24 +++
.../xtable/hudi/TestHudiConversionSource.java | 188 +++++++++++++++++++++
2 files changed, 212 insertions(+)
diff --git
a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionSource.java
b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionSource.java
index 00faa97d..4edcbed5 100644
--- a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionSource.java
+++ b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiConversionSource.java
@@ -39,6 +39,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.util.Option;
+import com.google.common.base.Strings;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
@@ -184,11 +185,34 @@ public class HudiConversionSource implements
ConversionSource<HoodieInstant> {
TimelineMetadataUtils.deserializeHoodieCleanMetadata(
metaClient.getActiveTimeline().getInstantDetails(lastCleanInstant.get()).get());
String earliestCommitToRetain = cleanMetadata.getEarliestCommitToRetain();
+ if (Strings.isNullOrEmpty(earliestCommitToRetain)) {
+ return cleanInstantsOccurredSinceLastSyncedInstant(instant);
+ }
Instant earliestCommitToRetainInstant =
HudiInstantUtils.parseFromInstantTime(earliestCommitToRetain);
return earliestCommitToRetainInstant.isAfter(instant);
}
+ // When clean instants have empty earliestCommitToRetain, trigger full
snapshot sync if any
+ // clean instants occurred after the last synced instant to err on the side
of caution
+ private boolean cleanInstantsOccurredSinceLastSyncedInstant(Instant instant)
{
+ String lastSyncedCommitTime =
HudiInstantUtils.convertInstantToCommit(instant);
+ List<HoodieInstant> cleanInstantsAfterLastSync =
+ metaClient
+ .getActiveTimeline()
+ .getCleanerTimeline()
+ .filterCompletedInstants()
+ .filter(
+ cleanInstant ->
+ HoodieTimeline.compareTimestamps(
+ cleanInstant.getTimestamp(),
+ HoodieTimeline.GREATER_THAN,
+ lastSyncedCommitTime))
+ .getInstants();
+
+ return !cleanInstantsAfterLastSync.isEmpty();
+ }
+
private CommitsPair getCompletedAndPendingCommitsForInstants(List<Instant>
lastPendingInstants) {
List<HoodieInstant> lastPendingHoodieInstants =
getCommitsForInstants(lastPendingInstants);
List<HoodieInstant> lastPendingHoodieInstantsCompleted =
diff --git
a/xtable-core/src/test/java/org/apache/xtable/hudi/TestHudiConversionSource.java
b/xtable-core/src/test/java/org/apache/xtable/hudi/TestHudiConversionSource.java
new file mode 100644
index 00000000..956fe29b
--- /dev/null
+++
b/xtable-core/src/test/java/org/apache/xtable/hudi/TestHudiConversionSource.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.hudi;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.jupiter.api.Test;
+
+import org.apache.hudi.avro.model.HoodieCleanMetadata;
+import org.apache.hudi.common.table.HoodieTableConfig;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.util.Option;
+
+class TestHudiConversionSource {
+
+ private static byte[] serializeCleanMetadata(String earliestCommitToRetain)
throws Exception {
+ HoodieCleanMetadata metadata =
+ HoodieCleanMetadata.newBuilder()
+ .setStartCleanTime("000")
+ .setTimeTakenInMillis(0L)
+ .setTotalFilesDeleted(0)
+ .setEarliestCommitToRetain(earliestCommitToRetain)
+ .setBootstrapPartitionMetadata(new HashMap<>())
+ .setPartitionMetadata(new HashMap<>())
+ .build();
+ return TimelineMetadataUtils.serializeCleanMetadata(metadata).get();
+ }
+
+ @Test
+ void
testIsIncrementalSyncSafeFromWithNullEarliestCommitToRetainNoCleanInstants()
+ throws Exception {
+ HoodieTableMetaClient mockMetaClient = mock(HoodieTableMetaClient.class);
+ HoodieActiveTimeline mockActiveTimeline = mock(HoodieActiveTimeline.class);
+ HoodieTimeline mockCleanerTimeline = mock(HoodieTimeline.class);
+ HoodieTimeline mockFilteredCleanerTimeline = mock(HoodieTimeline.class);
+ HoodieTimeline mockCompletedCommitsTimeline = mock(HoodieTimeline.class);
+ HoodieInstant mockCleanInstant = mock(HoodieInstant.class);
+ HoodieInstant mockCommitInstant = mock(HoodieInstant.class);
+
+ HoodieTableConfig mockTableConfig = mock(HoodieTableConfig.class);
+ when(mockMetaClient.getActiveTimeline()).thenReturn(mockActiveTimeline);
+ when(mockMetaClient.getHadoopConf()).thenReturn(new Configuration());
+ when(mockMetaClient.getTableConfig()).thenReturn(mockTableConfig);
+ when(mockTableConfig.isMetadataTableAvailable()).thenReturn(false);
+ when(mockMetaClient.getBasePathV2()).thenReturn(new
Path("/tmp/test-table"));
+
when(mockActiveTimeline.getCleanerTimeline()).thenReturn(mockCleanerTimeline);
+
when(mockCleanerTimeline.filterCompletedInstants()).thenReturn(mockCleanerTimeline);
+
when(mockCleanerTimeline.lastInstant()).thenReturn(Option.of(mockCleanInstant));
+ // Use empty string — Strings.isNullOrEmpty("") is true, same behavior as
null
+ when(mockActiveTimeline.getInstantDetails(mockCleanInstant))
+ .thenReturn(Option.of(serializeCleanMetadata("")));
+
+
when(mockCleanerTimeline.filter(any())).thenReturn(mockFilteredCleanerTimeline);
+
when(mockFilteredCleanerTimeline.getInstants()).thenReturn(Collections.emptyList());
+
+
when(mockActiveTimeline.filterCompletedInstants()).thenReturn(mockCompletedCommitsTimeline);
+
when(mockCompletedCommitsTimeline.findInstantsBeforeOrEquals(any(String.class)))
+ .thenReturn(mockCompletedCommitsTimeline);
+ when(mockCommitInstant.getTimestamp()).thenReturn("20200101120000000");
+
when(mockCompletedCommitsTimeline.lastInstant()).thenReturn(Option.of(mockCommitInstant));
+
+ HudiConversionSource hudiConversionSource =
+ new HudiConversionSource(mockMetaClient,
mock(PathBasedPartitionSpecExtractor.class));
+
+ Instant testInstant = Instant.now().minusSeconds(3600);
+ assertTrue(
+ hudiConversionSource.isIncrementalSyncSafeFrom(testInstant),
+ "isIncrementalSyncSafeFrom should return true when
earliestCommitToRetain is null and no clean instants after last sync");
+ }
+
+ @Test
+ void
testIsIncrementalSyncSafeFromWithNullEarliestCommitToRetainWithCleanInstants()
+ throws Exception {
+ HoodieTableMetaClient mockMetaClient = mock(HoodieTableMetaClient.class);
+ HoodieActiveTimeline mockActiveTimeline = mock(HoodieActiveTimeline.class);
+ HoodieTimeline mockCleanerTimeline = mock(HoodieTimeline.class);
+ HoodieTimeline mockFilteredCleanerTimeline = mock(HoodieTimeline.class);
+ HoodieTimeline mockCompletedCommitsTimeline = mock(HoodieTimeline.class);
+ HoodieInstant mockCleanInstant = mock(HoodieInstant.class);
+ HoodieInstant mockCleanInstantAfterSync = mock(HoodieInstant.class);
+ HoodieInstant mockCommitInstant = mock(HoodieInstant.class);
+
+ HoodieTableConfig mockTableConfig = mock(HoodieTableConfig.class);
+ when(mockMetaClient.getActiveTimeline()).thenReturn(mockActiveTimeline);
+ when(mockMetaClient.getHadoopConf()).thenReturn(new Configuration());
+ when(mockMetaClient.getTableConfig()).thenReturn(mockTableConfig);
+ when(mockTableConfig.isMetadataTableAvailable()).thenReturn(false);
+ when(mockMetaClient.getBasePathV2()).thenReturn(new
Path("/tmp/test-table"));
+
when(mockActiveTimeline.getCleanerTimeline()).thenReturn(mockCleanerTimeline);
+
when(mockCleanerTimeline.filterCompletedInstants()).thenReturn(mockCleanerTimeline);
+
when(mockCleanerTimeline.lastInstant()).thenReturn(Option.of(mockCleanInstant));
+ // Use empty string — Strings.isNullOrEmpty("") is true, same behavior as
null
+ when(mockActiveTimeline.getInstantDetails(mockCleanInstant))
+ .thenReturn(Option.of(serializeCleanMetadata("")));
+
+
when(mockCleanerTimeline.filter(any())).thenReturn(mockFilteredCleanerTimeline);
+ when(mockFilteredCleanerTimeline.getInstants())
+ .thenReturn(Arrays.asList(mockCleanInstantAfterSync));
+
+
when(mockActiveTimeline.filterCompletedInstants()).thenReturn(mockCompletedCommitsTimeline);
+
when(mockCompletedCommitsTimeline.findInstantsBeforeOrEquals(any(String.class)))
+ .thenReturn(mockCompletedCommitsTimeline);
+ when(mockCommitInstant.getTimestamp()).thenReturn("20200101120000000");
+
when(mockCompletedCommitsTimeline.lastInstant()).thenReturn(Option.of(mockCommitInstant));
+
+ HudiConversionSource hudiConversionSource =
+ new HudiConversionSource(mockMetaClient,
mock(PathBasedPartitionSpecExtractor.class));
+
+ Instant testInstant = Instant.now().minusSeconds(3600);
+ assertFalse(
+ hudiConversionSource.isIncrementalSyncSafeFrom(testInstant),
+ "isIncrementalSyncSafeFrom should return false when
earliestCommitToRetain is null but clean instants exist after last sync");
+ }
+
+ @Test
+ void
testIsIncrementalSyncSafeFromWithEmptyEarliestCommitToRetainWithCleanInstants()
+ throws Exception {
+ HoodieTableMetaClient mockMetaClient = mock(HoodieTableMetaClient.class);
+ HoodieActiveTimeline mockActiveTimeline = mock(HoodieActiveTimeline.class);
+ HoodieTimeline mockCleanerTimeline = mock(HoodieTimeline.class);
+ HoodieTimeline mockFilteredCleanerTimeline = mock(HoodieTimeline.class);
+ HoodieTimeline mockCompletedCommitsTimeline = mock(HoodieTimeline.class);
+ HoodieInstant mockCleanInstant = mock(HoodieInstant.class);
+ HoodieInstant mockCleanInstantAfterSync = mock(HoodieInstant.class);
+ HoodieInstant mockCommitInstant = mock(HoodieInstant.class);
+
+ HoodieTableConfig mockTableConfig = mock(HoodieTableConfig.class);
+ when(mockMetaClient.getActiveTimeline()).thenReturn(mockActiveTimeline);
+ when(mockMetaClient.getHadoopConf()).thenReturn(new Configuration());
+ when(mockMetaClient.getTableConfig()).thenReturn(mockTableConfig);
+ when(mockTableConfig.isMetadataTableAvailable()).thenReturn(false);
+ when(mockMetaClient.getBasePathV2()).thenReturn(new
Path("/tmp/test-table"));
+
when(mockActiveTimeline.getCleanerTimeline()).thenReturn(mockCleanerTimeline);
+
when(mockCleanerTimeline.filterCompletedInstants()).thenReturn(mockCleanerTimeline);
+
when(mockCleanerTimeline.lastInstant()).thenReturn(Option.of(mockCleanInstant));
+ when(mockActiveTimeline.getInstantDetails(mockCleanInstant))
+ .thenReturn(Option.of(serializeCleanMetadata("")));
+
+
when(mockCleanerTimeline.filter(any())).thenReturn(mockFilteredCleanerTimeline);
+ when(mockFilteredCleanerTimeline.getInstants())
+ .thenReturn(Arrays.asList(mockCleanInstantAfterSync));
+
+
when(mockActiveTimeline.filterCompletedInstants()).thenReturn(mockCompletedCommitsTimeline);
+
when(mockCompletedCommitsTimeline.findInstantsBeforeOrEquals(any(String.class)))
+ .thenReturn(mockCompletedCommitsTimeline);
+ when(mockCommitInstant.getTimestamp()).thenReturn("20200101120000000");
+
when(mockCompletedCommitsTimeline.lastInstant()).thenReturn(Option.of(mockCommitInstant));
+
+ HudiConversionSource hudiConversionSource =
+ new HudiConversionSource(mockMetaClient,
mock(PathBasedPartitionSpecExtractor.class));
+
+ Instant testInstant = Instant.now().minusSeconds(3600);
+ assertFalse(
+ hudiConversionSource.isIncrementalSyncSafeFrom(testInstant),
+ "isIncrementalSyncSafeFrom should return false when
earliestCommitToRetain is empty and clean instants exist after last sync");
+ }
+}