This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 65c9034e4fe1 fix(meta-sync): bound the dropped-partition clean scan to
the last commit time synced (#20380)
65c9034e4fe1 is described below
commit 65c9034e4fe192832aa62e0ee552e85192f165d9
Author: Y Ethan Guo <[email protected]>
AuthorDate: Wed Nov 18 07:48:43 2026 +0000
fix(meta-sync): bound the dropped-partition clean scan to the last commit
time synced (#20380)
Meta sync read every completed clean on the active timeline to detect
dropped partitions, so on a lagging table it raced the archiver over
cleans whose partition deletions earlier syncs had already applied, and
failed for the cycle when one disappeared mid-read.
The clean scan now covers only the cleans since the last sync (the same
window as the write scan) when both watermarks are known, and a clean
inside that window that cannot be read fails the call instead of being
skipped. When a timeline read fails because the archiver removed instants
the sync still needed, HiveSyncTool falls back to listing all partitions
in the same sync, without replacing the timeline it works from.
---
.../hudi/common/table/timeline/TimelineUtils.java | 28 +++++-
.../hudi/common/table/TestTimelineUtils.java | 75 +++++++++++++++
.../java/org/apache/hudi/hive/HiveSyncTool.java | 43 +++++++--
.../TestHiveSyncToolPartitionSyncFallback.java | 107 +++++++++++++++++++++
4 files changed, 240 insertions(+), 13 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java
index fa87f85303da..733495033f58 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java
@@ -103,6 +103,8 @@ public class TimelineUtils {
/**
* Returns partitions that have been deleted or marked for deletion in the
timeline between given commit time range.
* Does not include internal operations such as clean in the timeline.
+ *
+ * @throws HoodieIOException if an instant since the last sync, including a
clean, cannot be read
*/
public static List<String> getDroppedPartitions(HoodieTableMetaClient
metaClient, Option<String> lastCommitTimeSynced, Option<String>
lastCommitCompletionTimeSynced) {
HoodieTimeline timeline = lastCommitTimeSynced.isPresent()
@@ -125,7 +127,13 @@ public class TimelineUtils {
.map(partition -> new AbstractMap.SimpleEntry<>(partition,
pair.getLeft().requestedTime()))
).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue,
(existing, replace) -> replace));
// cleaner could delete a partition when there are no active filegroups in
the partition
- HoodieTimeline cleanerTimeline =
metaClient.getActiveTimeline().getCleanerTimeline().filterCompletedInstants();
+ HoodieTimeline completedCleanerTimeline =
metaClient.getActiveTimeline().getCleanerTimeline().filterCompletedInstants();
+ // Both watermarks are needed to bound the scan: without the completion
time, a clean requested before the last
+ // commit time synced but still inflight at the last sync would never be
read.
+ boolean cleansSinceLastSyncOnly = lastCommitTimeSynced.isPresent() &&
lastCommitCompletionTimeSynced.isPresent();
+ HoodieTimeline cleanerTimeline = cleansSinceLastSyncOnly
+ ? getInstantsSince(completedCleanerTimeline,
lastCommitTimeSynced.get(), lastCommitCompletionTimeSynced)
+ : completedCleanerTimeline;
cleanerTimeline.getInstantsAsStream()
.forEach(instant -> {
try {
@@ -136,7 +144,9 @@ public class TimelineUtils {
}
});
} catch (HoodieIOException e) {
- if (e.getCause() instanceof FileNotFoundException) {
+ // The unbounded scan is best effort over every active clean, so
one the archiver removed mid-scan is
+ // skipped; inside the window every clean is required.
+ if (!cleansSinceLastSyncOnly && e.getCause() instanceof
FileNotFoundException) {
log.warn("Instant {} not found in storage and has been
archived", instant, e);
} else {
throw e;
@@ -323,12 +333,20 @@ public class TimelineUtils {
?
metaClient.getArchivedTimeline(exclusiveStartInstantTime).mergeTimeline(writeTimeline)
: writeTimeline;
- HoodieTimeline timelineSinceLastSync = timeline.getCommitsTimeline()
- .findInstantsAfter(exclusiveStartInstantTime, Integer.MAX_VALUE);
+ return getInstantsSince(timeline.getCommitsTimeline(),
exclusiveStartInstantTime, lastMaxCompletionTime);
+ }
+
+ /**
+ * Returns the instants of the timeline that a sync up to {@code
exclusiveStartInstantTime} could not have seen:
+ * those requested after it, plus the completed "hollow" ones requested
before it but completed after
+ * {@code lastMaxCompletionTime}.
+ */
+ public static HoodieTimeline getInstantsSince(HoodieTimeline timeline,
String exclusiveStartInstantTime, Option<String> lastMaxCompletionTime) {
+ HoodieTimeline timelineSinceLastSync =
timeline.findInstantsAfter(exclusiveStartInstantTime, Integer.MAX_VALUE);
if (lastMaxCompletionTime.isPresent()) {
// Get 'hollow' instants that have less instant time than
exclusiveStartInstantTime but with greater commit completion time
- HoodieTimeline hollowInstantsTimeline = timeline.getCommitsTimeline()
+ HoodieTimeline hollowInstantsTimeline = timeline
.filterCompletedInstants()
.filter(s -> compareTimestamps(s.requestedTime(), LESSER_THAN,
exclusiveStartInstantTime))
.filter(s -> compareTimestamps(s.getCompletionTime(), GREATER_THAN,
lastMaxCompletionTime.get()));
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java
index 9472cc7cc3cd..7be12fdfd1b6 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java
@@ -52,6 +52,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
+import java.io.OutputStream;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
@@ -61,6 +62,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static
org.apache.hudi.common.table.timeline.HoodieInstant.State.COMPLETED;
import static
org.apache.hudi.common.table.timeline.HoodieInstant.State.INFLIGHT;
@@ -664,6 +666,79 @@ class TestTimelineUtils extends HoodieCommonTestHarness {
assertTrue(droppedPartitions.isEmpty());
}
+ /**
+ * With a last commit time synced, only the cleans since it are read, and a
clean in that window that cannot be
+ * read fails the call instead of being skipped. Runs on both timeline
layouts, since layout V1 (table version 6)
+ * derives completion times from file modification times rather than the
instant file name.
+ */
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void
testGetDroppedPartitionsReadsOnlyCleansSinceLastCommitTimeSynced(boolean
preTableVersion8) throws Exception {
+ initMetaClient(preTableVersion8);
+ assertEquals(preTableVersion8 ? HoodieTableVersion.SIX :
HoodieTableVersion.current(), metaClient.getTableConfig().getTableVersion());
+ HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
+ String olderPartition = "p1";
+ HoodieInstant instant1 = new HoodieInstant(INFLIGHT, COMMIT_ACTION,
"00001", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR);
+ activeTimeline.createNewInstant(instant1);
+ activeTimeline.saveAsComplete(instant1, getCommitMetadata(basePath,
olderPartition, "00001", 2, Collections.emptyMap()));
+ HoodieInstant instant2 = new HoodieInstant(INFLIGHT, COMMIT_ACTION,
"00002", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR);
+ activeTimeline.createNewInstant(instant2);
+ activeTimeline.saveAsComplete(instant2, getCommitMetadata(basePath, "p2",
"00002", 2, Collections.emptyMap()));
+ HoodieInstant cleanInstant = new HoodieInstant(INFLIGHT, CLEAN_ACTION,
"00003", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR);
+ activeTimeline.createNewInstant(cleanInstant);
+ activeTimeline.saveAsComplete(cleanInstant,
getCleanMetadata(olderPartition, "00003", true));
+ metaClient.reloadActiveTimeline();
+
+ HoodieInstant completedClean =
metaClient.getActiveTimeline().getCleanerTimeline().filterCompletedInstants().lastInstant().get();
+ Option<String> completionTimeSynced =
Option.of(completedClean.getCompletionTime());
+
+ // The clean is after the last commit time synced, so it is read and
reports the drop.
+ assertEquals(Collections.singletonList(olderPartition),
+ TimelineUtils.getDroppedPartitions(metaClient, Option.of("00002"),
completionTimeSynced));
+
+ // Corrupt the clean file: inside the window it is still read, and the
failure surfaces.
+ StoragePath cleanPath =
metaClient.getStorage().listDirectEntries(metaClient.getTimelinePath()).stream()
+ .map(pathInfo -> pathInfo.getPath())
+ .filter(path -> path.getName().endsWith(CLEAN_ACTION))
+ .findFirst().get();
+ try (OutputStream out = metaClient.getStorage().create(cleanPath, true)) {
+ out.write("not an avro file".getBytes());
+ }
+ HoodieIOException corrupt = assertThrows(HoodieIOException.class,
+ () -> TimelineUtils.getDroppedPartitions(metaClient,
Option.of("00002"), completionTimeSynced));
+ assertTrue(corrupt.getMessage().startsWith("Failed to get partitions
cleaned at"), corrupt.getMessage());
+
+ // The clean is neither after the last commit time synced nor completed
after the completion time synced,
+ // so it is not read at all.
+ assertEquals(Collections.emptyList(),
+ TimelineUtils.getDroppedPartitions(metaClient, Option.of("00003"),
completionTimeSynced));
+ // Without the completion time synced the window cannot be trusted, so
every active clean is read.
+ assertThrows(HoodieIOException.class,
+ () -> TimelineUtils.getDroppedPartitions(metaClient,
Option.of("00003"), Option.empty()));
+
+ // Missing inside the window: the drop cannot be known, so the call fails.
+ metaClient.getStorage().deleteFile(cleanPath);
+ assertThrows(HoodieIOException.class,
+ () -> TimelineUtils.getDroppedPartitions(metaClient,
Option.of("00002"), completionTimeSynced));
+ // Missing without a last commit time synced: skipped, as the whole active
timeline is scanned best effort.
+ assertEquals(Collections.emptyList(),
+ TimelineUtils.getDroppedPartitions(metaClient, Option.empty(),
Option.empty()));
+ }
+
+ @Test
+ void testGetInstantsSince() {
+ HoodieInstant olderClean = new HoodieInstant(COMPLETED, CLEAN_ACTION,
"001", "002", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR);
+ // Requested before the last commit time synced but completed after the
last completion time synced.
+ HoodieInstant hollowClean = new HoodieInstant(COMPLETED, CLEAN_ACTION,
"003", "012", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR);
+ HoodieInstant inflightClean = new HoodieInstant(INFLIGHT, CLEAN_ACTION,
"004", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR);
+ HoodieInstant newerClean = new HoodieInstant(COMPLETED, CLEAN_ACTION,
"011", "013", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR);
+ HoodieTimeline cleans = new BaseTimelineV2(Stream.of(olderClean,
hollowClean, inflightClean, newerClean), null);
+
+ verifyTimeline(Collections.singletonList(newerClean),
TimelineUtils.getInstantsSince(cleans, "010", Option.empty()));
+ verifyTimeline(Arrays.asList(hollowClean, newerClean),
TimelineUtils.getInstantsSince(cleans, "010", Option.of("011")));
+ verifyTimeline(Collections.singletonList(newerClean),
TimelineUtils.getInstantsSince(cleans, "010", Option.of("012")));
+ }
+
@Test
void testGetLastCommitMetadataWithSchemaIgnoresOperationType() throws
Exception {
HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
index c45f3d1e79c9..4ea02b177a61 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
@@ -28,6 +28,7 @@ import org.apache.hudi.common.util.ConfigUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.InvalidTableException;
import org.apache.hudi.sync.common.HoodieSyncClient;
import org.apache.hudi.sync.common.HoodieSyncTool;
@@ -44,6 +45,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import java.io.FileNotFoundException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -382,7 +384,8 @@ public class HiveSyncTool extends HoodieSyncTool implements
AutoCloseable {
}
}
- private boolean validateAndSyncPartitions(String tableName, boolean
tableExists) {
+ @VisibleForTesting
+ boolean validateAndSyncPartitions(String tableName, boolean tableExists) {
boolean syncIncremental = isIncrementalSync();
Option<String> lastCommitTimeSynced = (tableExists && syncIncremental)
? syncClient.getLastCommitTimeSynced(tableName) : Option.empty();
@@ -408,20 +411,44 @@ public class HiveSyncTool extends HoodieSyncTool
implements AutoCloseable {
config.getString(META_SYNC_BASE_PATH), config.getHadoopFileSystem());
partitionsChanged = syncAllPartitions(tableName);
} else {
- List<String> writtenPartitionsSince = syncMetrics.timePartitionScan(
- () -> syncClient.getWrittenPartitionsSince(lastCommitTimeSynced,
lastCommitCompletionTimeSynced));
+ List<String> writtenPartitionsSince;
+ Set<String> droppedPartitions;
+ try {
+ writtenPartitionsSince = syncMetrics.timePartitionScan(
+ () -> syncClient.getWrittenPartitionsSince(lastCommitTimeSynced,
lastCommitCompletionTimeSynced));
+ // find dropped partitions, if any, in the latest commit
+ droppedPartitions = syncMetrics.timePartitionScan(
+ () -> syncClient.getDroppedPartitionsSince(lastCommitTimeSynced,
lastCommitCompletionTimeSynced));
+ } catch (HoodieIOException e) {
+ if (!hasFileNotFoundCause(e)) {
+ throw e;
+ }
+ log.warn("Instants since last commit time synced {} were archived
while reading the timeline. Listing all partitions in {}",
+ lastCommitTimeSynced.get(), config.getString(META_SYNC_BASE_PATH),
e);
+ return syncAllPartitions(tableName);
+ }
log.info("Storage partitions scan complete. Found {}",
writtenPartitionsSince.size());
-
- // Sync the partitions if needed
- // find dropped partitions, if any, in the latest commit
- Set<String> droppedPartitions = syncMetrics.timePartitionScan(
- () -> syncClient.getDroppedPartitionsSince(lastCommitTimeSynced,
lastCommitCompletionTimeSynced));
log.info("Partitions dropped since last sync: {}",
droppedPartitions.size());
+ // Sync the partitions if needed
partitionsChanged = syncPartitions(tableName, writtenPartitionsSince,
droppedPartitions);
}
return partitionsChanged;
}
+ /**
+ * A timeline read that fails because the archiver removed the instant file
carries a FileNotFoundException in
+ * its cause chain, directly from the open or wrapped by the Avro or JSON
reader on a lazy read. Listing all
+ * partitions on storage is then the right answer; any other read failure is
a real error.
+ */
+ private static boolean hasFileNotFoundCause(Throwable throwable) {
+ for (Throwable cause = throwable; cause != null; cause = cause.getCause())
{
+ if (cause instanceof FileNotFoundException) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private boolean isIncrementalSync() {
return config.getBoolean(META_SYNC_INCREMENTAL);
}
diff --git
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolPartitionSyncFallback.java
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolPartitionSyncFallback.java
new file mode 100644
index 000000000000..017f60ba745f
--- /dev/null
+++
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolPartitionSyncFallback.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.hive;
+
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.testutils.MockHoodieTimeline;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.sync.common.HoodieSyncClient;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.stream.Stream;
+
+import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_INCREMENTAL;
+import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests the incremental partition sync falling back to a full listing, within
the same sync, when a timeline read
+ * fails because the archiver removed an instant the sync still needed.
+ */
+class TestHiveSyncToolPartitionSyncFallback {
+
+ private static final String TABLE_NAME = "table";
+ private static final String LAST_COMMIT_TIME_SYNCED = "102";
+
+ @Test
+ void syncsAllPartitionsWhenAnInstantSinceLastSyncIsArchived() throws
ReflectiveOperationException {
+ HoodieSyncClient syncClient = mockSyncClient();
+ when(syncClient.getWrittenPartitionsSince(any(),
any())).thenReturn(Collections.singletonList("p"));
+ // The shape of a lazy S3 read of a deleted instant: the
FileNotFoundException is two causes deep.
+ when(syncClient.getDroppedPartitionsSince(any(), any())).thenThrow(new
HoodieIOException("Failed to get partitions cleaned at 103",
+ new IOException("Not an Avro data file.", new
FileNotFoundException("103.clean"))));
+ HiveSyncTool tool = toolWith(syncClient);
+
+ assertFalse(tool.validateAndSyncPartitions(TABLE_NAME, true));
+
+ verify(syncClient).getAllPartitionPathsOnStorage();
+ // The timeline this sync works from is left alone, so the last commit
time synced recorded later matches it.
+ verify(syncClient.getMetaClient(), never()).reloadActiveTimeline();
+ }
+
+ @Test
+ void rethrowsAReadFailureThatIsNotAMissingInstant() throws
ReflectiveOperationException {
+ HoodieSyncClient syncClient = mockSyncClient();
+ when(syncClient.getWrittenPartitionsSince(any(),
any())).thenReturn(Collections.singletonList("p"));
+ HoodieIOException readFailure = new HoodieIOException("Failed to get
partitions cleaned at 103", new IOException("Not an Avro data file."));
+ when(syncClient.getDroppedPartitionsSince(any(),
any())).thenThrow(readFailure);
+ HiveSyncTool tool = toolWith(syncClient);
+
+ assertSame(readFailure, assertThrows(HoodieIOException.class, () ->
tool.validateAndSyncPartitions(TABLE_NAME, true)));
+
+ verify(syncClient, never()).getAllPartitionPathsOnStorage();
+ }
+
+ private static HoodieSyncClient mockSyncClient() {
+ HoodieSyncClient syncClient = mock(HoodieSyncClient.class);
+
when(syncClient.getMetaClient()).thenReturn(mock(HoodieTableMetaClient.class));
+ when(syncClient.getActiveTimeline()).thenReturn(new
MockHoodieTimeline(Stream.of("100", "102", "104"), Stream.empty()));
+
when(syncClient.getLastCommitTimeSynced(TABLE_NAME)).thenReturn(Option.of(LAST_COMMIT_TIME_SYNCED));
+
when(syncClient.getLastCommitCompletionTimeSynced(TABLE_NAME)).thenReturn(Option.empty());
+ return syncClient;
+ }
+
+ private static HiveSyncTool toolWith(HoodieSyncClient syncClient) throws
ReflectiveOperationException {
+ HiveSyncConfig config = mock(HiveSyncConfig.class);
+ when(config.getBoolean(META_SYNC_INCREMENTAL)).thenReturn(true);
+
when(config.getSplitStrings(META_SYNC_PARTITION_FIELDS)).thenReturn(Collections.singletonList("p"));
+ HiveSyncTool tool = mock(HiveSyncTool.class, CALLS_REAL_METHODS);
+ tool.config = config;
+ tool.syncClient = syncClient;
+ // The mock skips the constructor, which is what normally creates the sync
metrics the partition scan is timed with.
+ Field syncMetrics = HiveSyncTool.class.getDeclaredField("syncMetrics");
+ syncMetrics.setAccessible(true);
+ syncMetrics.set(tool, new HoodieHiveSyncStats(false));
+ return tool;
+ }
+}