This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20380-6a0c435db649fce65348958f535f1039a9d35c10 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 65c9034e4fe192832aa62e0ee552e85192f165d9 Author: Y Ethan Guo <[email protected]> AuthorDate: Wed Nov 18 07:48:43 2026 +0000 fix(meta-sync): bound the dropped-partition clean scan to the last commit time synced (#20380) Meta sync read every completed clean on the active timeline to detect dropped partitions, so on a lagging table it raced the archiver over cleans whose partition deletions earlier syncs had already applied, and failed for the cycle when one disappeared mid-read. The clean scan now covers only the cleans since the last sync (the same window as the write scan) when both watermarks are known, and a clean inside that window that cannot be read fails the call instead of being skipped. When a timeline read fails because the archiver removed instants the sync still needed, HiveSyncTool falls back to listing all partitions in the same sync, without replacing the timeline it works from. --- .../hudi/common/table/timeline/TimelineUtils.java | 28 +++++- .../hudi/common/table/TestTimelineUtils.java | 75 +++++++++++++++ .../java/org/apache/hudi/hive/HiveSyncTool.java | 43 +++++++-- .../TestHiveSyncToolPartitionSyncFallback.java | 107 +++++++++++++++++++++ 4 files changed, 240 insertions(+), 13 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java index fa87f85303da..733495033f58 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineUtils.java @@ -103,6 +103,8 @@ public class TimelineUtils { /** * Returns partitions that have been deleted or marked for deletion in the timeline between given commit time range. * Does not include internal operations such as clean in the timeline. + * + * @throws HoodieIOException if an instant since the last sync, including a clean, cannot be read */ public static List<String> getDroppedPartitions(HoodieTableMetaClient metaClient, Option<String> lastCommitTimeSynced, Option<String> lastCommitCompletionTimeSynced) { HoodieTimeline timeline = lastCommitTimeSynced.isPresent() @@ -125,7 +127,13 @@ public class TimelineUtils { .map(partition -> new AbstractMap.SimpleEntry<>(partition, pair.getLeft().requestedTime())) ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (existing, replace) -> replace)); // cleaner could delete a partition when there are no active filegroups in the partition - HoodieTimeline cleanerTimeline = metaClient.getActiveTimeline().getCleanerTimeline().filterCompletedInstants(); + HoodieTimeline completedCleanerTimeline = metaClient.getActiveTimeline().getCleanerTimeline().filterCompletedInstants(); + // Both watermarks are needed to bound the scan: without the completion time, a clean requested before the last + // commit time synced but still inflight at the last sync would never be read. + boolean cleansSinceLastSyncOnly = lastCommitTimeSynced.isPresent() && lastCommitCompletionTimeSynced.isPresent(); + HoodieTimeline cleanerTimeline = cleansSinceLastSyncOnly + ? getInstantsSince(completedCleanerTimeline, lastCommitTimeSynced.get(), lastCommitCompletionTimeSynced) + : completedCleanerTimeline; cleanerTimeline.getInstantsAsStream() .forEach(instant -> { try { @@ -136,7 +144,9 @@ public class TimelineUtils { } }); } catch (HoodieIOException e) { - if (e.getCause() instanceof FileNotFoundException) { + // The unbounded scan is best effort over every active clean, so one the archiver removed mid-scan is + // skipped; inside the window every clean is required. + if (!cleansSinceLastSyncOnly && e.getCause() instanceof FileNotFoundException) { log.warn("Instant {} not found in storage and has been archived", instant, e); } else { throw e; @@ -323,12 +333,20 @@ public class TimelineUtils { ? metaClient.getArchivedTimeline(exclusiveStartInstantTime).mergeTimeline(writeTimeline) : writeTimeline; - HoodieTimeline timelineSinceLastSync = timeline.getCommitsTimeline() - .findInstantsAfter(exclusiveStartInstantTime, Integer.MAX_VALUE); + return getInstantsSince(timeline.getCommitsTimeline(), exclusiveStartInstantTime, lastMaxCompletionTime); + } + + /** + * Returns the instants of the timeline that a sync up to {@code exclusiveStartInstantTime} could not have seen: + * those requested after it, plus the completed "hollow" ones requested before it but completed after + * {@code lastMaxCompletionTime}. + */ + public static HoodieTimeline getInstantsSince(HoodieTimeline timeline, String exclusiveStartInstantTime, Option<String> lastMaxCompletionTime) { + HoodieTimeline timelineSinceLastSync = timeline.findInstantsAfter(exclusiveStartInstantTime, Integer.MAX_VALUE); if (lastMaxCompletionTime.isPresent()) { // Get 'hollow' instants that have less instant time than exclusiveStartInstantTime but with greater commit completion time - HoodieTimeline hollowInstantsTimeline = timeline.getCommitsTimeline() + HoodieTimeline hollowInstantsTimeline = timeline .filterCompletedInstants() .filter(s -> compareTimestamps(s.requestedTime(), LESSER_THAN, exclusiveStartInstantTime)) .filter(s -> compareTimestamps(s.getCompletionTime(), GREATER_THAN, lastMaxCompletionTime.get())); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java index 9472cc7cc3cd..7be12fdfd1b6 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestTimelineUtils.java @@ -52,6 +52,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.io.OutputStream; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -61,6 +62,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.hudi.common.table.timeline.HoodieInstant.State.COMPLETED; import static org.apache.hudi.common.table.timeline.HoodieInstant.State.INFLIGHT; @@ -664,6 +666,79 @@ class TestTimelineUtils extends HoodieCommonTestHarness { assertTrue(droppedPartitions.isEmpty()); } + /** + * With a last commit time synced, only the cleans since it are read, and a clean in that window that cannot be + * read fails the call instead of being skipped. Runs on both timeline layouts, since layout V1 (table version 6) + * derives completion times from file modification times rather than the instant file name. + */ + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testGetDroppedPartitionsReadsOnlyCleansSinceLastCommitTimeSynced(boolean preTableVersion8) throws Exception { + initMetaClient(preTableVersion8); + assertEquals(preTableVersion8 ? HoodieTableVersion.SIX : HoodieTableVersion.current(), metaClient.getTableConfig().getTableVersion()); + HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline(); + String olderPartition = "p1"; + HoodieInstant instant1 = new HoodieInstant(INFLIGHT, COMMIT_ACTION, "00001", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR); + activeTimeline.createNewInstant(instant1); + activeTimeline.saveAsComplete(instant1, getCommitMetadata(basePath, olderPartition, "00001", 2, Collections.emptyMap())); + HoodieInstant instant2 = new HoodieInstant(INFLIGHT, COMMIT_ACTION, "00002", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR); + activeTimeline.createNewInstant(instant2); + activeTimeline.saveAsComplete(instant2, getCommitMetadata(basePath, "p2", "00002", 2, Collections.emptyMap())); + HoodieInstant cleanInstant = new HoodieInstant(INFLIGHT, CLEAN_ACTION, "00003", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR); + activeTimeline.createNewInstant(cleanInstant); + activeTimeline.saveAsComplete(cleanInstant, getCleanMetadata(olderPartition, "00003", true)); + metaClient.reloadActiveTimeline(); + + HoodieInstant completedClean = metaClient.getActiveTimeline().getCleanerTimeline().filterCompletedInstants().lastInstant().get(); + Option<String> completionTimeSynced = Option.of(completedClean.getCompletionTime()); + + // The clean is after the last commit time synced, so it is read and reports the drop. + assertEquals(Collections.singletonList(olderPartition), + TimelineUtils.getDroppedPartitions(metaClient, Option.of("00002"), completionTimeSynced)); + + // Corrupt the clean file: inside the window it is still read, and the failure surfaces. + StoragePath cleanPath = metaClient.getStorage().listDirectEntries(metaClient.getTimelinePath()).stream() + .map(pathInfo -> pathInfo.getPath()) + .filter(path -> path.getName().endsWith(CLEAN_ACTION)) + .findFirst().get(); + try (OutputStream out = metaClient.getStorage().create(cleanPath, true)) { + out.write("not an avro file".getBytes()); + } + HoodieIOException corrupt = assertThrows(HoodieIOException.class, + () -> TimelineUtils.getDroppedPartitions(metaClient, Option.of("00002"), completionTimeSynced)); + assertTrue(corrupt.getMessage().startsWith("Failed to get partitions cleaned at"), corrupt.getMessage()); + + // The clean is neither after the last commit time synced nor completed after the completion time synced, + // so it is not read at all. + assertEquals(Collections.emptyList(), + TimelineUtils.getDroppedPartitions(metaClient, Option.of("00003"), completionTimeSynced)); + // Without the completion time synced the window cannot be trusted, so every active clean is read. + assertThrows(HoodieIOException.class, + () -> TimelineUtils.getDroppedPartitions(metaClient, Option.of("00003"), Option.empty())); + + // Missing inside the window: the drop cannot be known, so the call fails. + metaClient.getStorage().deleteFile(cleanPath); + assertThrows(HoodieIOException.class, + () -> TimelineUtils.getDroppedPartitions(metaClient, Option.of("00002"), completionTimeSynced)); + // Missing without a last commit time synced: skipped, as the whole active timeline is scanned best effort. + assertEquals(Collections.emptyList(), + TimelineUtils.getDroppedPartitions(metaClient, Option.empty(), Option.empty())); + } + + @Test + void testGetInstantsSince() { + HoodieInstant olderClean = new HoodieInstant(COMPLETED, CLEAN_ACTION, "001", "002", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR); + // Requested before the last commit time synced but completed after the last completion time synced. + HoodieInstant hollowClean = new HoodieInstant(COMPLETED, CLEAN_ACTION, "003", "012", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR); + HoodieInstant inflightClean = new HoodieInstant(INFLIGHT, CLEAN_ACTION, "004", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR); + HoodieInstant newerClean = new HoodieInstant(COMPLETED, CLEAN_ACTION, "011", "013", InstantComparatorV2.REQUESTED_TIME_BASED_COMPARATOR); + HoodieTimeline cleans = new BaseTimelineV2(Stream.of(olderClean, hollowClean, inflightClean, newerClean), null); + + verifyTimeline(Collections.singletonList(newerClean), TimelineUtils.getInstantsSince(cleans, "010", Option.empty())); + verifyTimeline(Arrays.asList(hollowClean, newerClean), TimelineUtils.getInstantsSince(cleans, "010", Option.of("011"))); + verifyTimeline(Collections.singletonList(newerClean), TimelineUtils.getInstantsSince(cleans, "010", Option.of("012"))); + } + @Test void testGetLastCommitMetadataWithSchemaIgnoresOperationType() throws Exception { HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline(); diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java index c45f3d1e79c9..4ea02b177a61 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java @@ -28,6 +28,7 @@ import org.apache.hudi.common.util.ConfigUtils; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.VisibleForTesting; import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; import org.apache.hudi.exception.InvalidTableException; import org.apache.hudi.sync.common.HoodieSyncClient; import org.apache.hudi.sync.common.HoodieSyncTool; @@ -44,6 +45,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import java.io.FileNotFoundException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -382,7 +384,8 @@ public class HiveSyncTool extends HoodieSyncTool implements AutoCloseable { } } - private boolean validateAndSyncPartitions(String tableName, boolean tableExists) { + @VisibleForTesting + boolean validateAndSyncPartitions(String tableName, boolean tableExists) { boolean syncIncremental = isIncrementalSync(); Option<String> lastCommitTimeSynced = (tableExists && syncIncremental) ? syncClient.getLastCommitTimeSynced(tableName) : Option.empty(); @@ -408,20 +411,44 @@ public class HiveSyncTool extends HoodieSyncTool implements AutoCloseable { config.getString(META_SYNC_BASE_PATH), config.getHadoopFileSystem()); partitionsChanged = syncAllPartitions(tableName); } else { - List<String> writtenPartitionsSince = syncMetrics.timePartitionScan( - () -> syncClient.getWrittenPartitionsSince(lastCommitTimeSynced, lastCommitCompletionTimeSynced)); + List<String> writtenPartitionsSince; + Set<String> droppedPartitions; + try { + writtenPartitionsSince = syncMetrics.timePartitionScan( + () -> syncClient.getWrittenPartitionsSince(lastCommitTimeSynced, lastCommitCompletionTimeSynced)); + // find dropped partitions, if any, in the latest commit + droppedPartitions = syncMetrics.timePartitionScan( + () -> syncClient.getDroppedPartitionsSince(lastCommitTimeSynced, lastCommitCompletionTimeSynced)); + } catch (HoodieIOException e) { + if (!hasFileNotFoundCause(e)) { + throw e; + } + log.warn("Instants since last commit time synced {} were archived while reading the timeline. Listing all partitions in {}", + lastCommitTimeSynced.get(), config.getString(META_SYNC_BASE_PATH), e); + return syncAllPartitions(tableName); + } log.info("Storage partitions scan complete. Found {}", writtenPartitionsSince.size()); - - // Sync the partitions if needed - // find dropped partitions, if any, in the latest commit - Set<String> droppedPartitions = syncMetrics.timePartitionScan( - () -> syncClient.getDroppedPartitionsSince(lastCommitTimeSynced, lastCommitCompletionTimeSynced)); log.info("Partitions dropped since last sync: {}", droppedPartitions.size()); + // Sync the partitions if needed partitionsChanged = syncPartitions(tableName, writtenPartitionsSince, droppedPartitions); } return partitionsChanged; } + /** + * A timeline read that fails because the archiver removed the instant file carries a FileNotFoundException in + * its cause chain, directly from the open or wrapped by the Avro or JSON reader on a lazy read. Listing all + * partitions on storage is then the right answer; any other read failure is a real error. + */ + private static boolean hasFileNotFoundCause(Throwable throwable) { + for (Throwable cause = throwable; cause != null; cause = cause.getCause()) { + if (cause instanceof FileNotFoundException) { + return true; + } + } + return false; + } + private boolean isIncrementalSync() { return config.getBoolean(META_SYNC_INCREMENTAL); } diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolPartitionSyncFallback.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolPartitionSyncFallback.java new file mode 100644 index 000000000000..017f60ba745f --- /dev/null +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncToolPartitionSyncFallback.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.hive; + +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.MockHoodieTimeline; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.sync.common.HoodieSyncClient; + +import org.junit.jupiter.api.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.stream.Stream; + +import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_INCREMENTAL; +import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Tests the incremental partition sync falling back to a full listing, within the same sync, when a timeline read + * fails because the archiver removed an instant the sync still needed. + */ +class TestHiveSyncToolPartitionSyncFallback { + + private static final String TABLE_NAME = "table"; + private static final String LAST_COMMIT_TIME_SYNCED = "102"; + + @Test + void syncsAllPartitionsWhenAnInstantSinceLastSyncIsArchived() throws ReflectiveOperationException { + HoodieSyncClient syncClient = mockSyncClient(); + when(syncClient.getWrittenPartitionsSince(any(), any())).thenReturn(Collections.singletonList("p")); + // The shape of a lazy S3 read of a deleted instant: the FileNotFoundException is two causes deep. + when(syncClient.getDroppedPartitionsSince(any(), any())).thenThrow(new HoodieIOException("Failed to get partitions cleaned at 103", + new IOException("Not an Avro data file.", new FileNotFoundException("103.clean")))); + HiveSyncTool tool = toolWith(syncClient); + + assertFalse(tool.validateAndSyncPartitions(TABLE_NAME, true)); + + verify(syncClient).getAllPartitionPathsOnStorage(); + // The timeline this sync works from is left alone, so the last commit time synced recorded later matches it. + verify(syncClient.getMetaClient(), never()).reloadActiveTimeline(); + } + + @Test + void rethrowsAReadFailureThatIsNotAMissingInstant() throws ReflectiveOperationException { + HoodieSyncClient syncClient = mockSyncClient(); + when(syncClient.getWrittenPartitionsSince(any(), any())).thenReturn(Collections.singletonList("p")); + HoodieIOException readFailure = new HoodieIOException("Failed to get partitions cleaned at 103", new IOException("Not an Avro data file.")); + when(syncClient.getDroppedPartitionsSince(any(), any())).thenThrow(readFailure); + HiveSyncTool tool = toolWith(syncClient); + + assertSame(readFailure, assertThrows(HoodieIOException.class, () -> tool.validateAndSyncPartitions(TABLE_NAME, true))); + + verify(syncClient, never()).getAllPartitionPathsOnStorage(); + } + + private static HoodieSyncClient mockSyncClient() { + HoodieSyncClient syncClient = mock(HoodieSyncClient.class); + when(syncClient.getMetaClient()).thenReturn(mock(HoodieTableMetaClient.class)); + when(syncClient.getActiveTimeline()).thenReturn(new MockHoodieTimeline(Stream.of("100", "102", "104"), Stream.empty())); + when(syncClient.getLastCommitTimeSynced(TABLE_NAME)).thenReturn(Option.of(LAST_COMMIT_TIME_SYNCED)); + when(syncClient.getLastCommitCompletionTimeSynced(TABLE_NAME)).thenReturn(Option.empty()); + return syncClient; + } + + private static HiveSyncTool toolWith(HoodieSyncClient syncClient) throws ReflectiveOperationException { + HiveSyncConfig config = mock(HiveSyncConfig.class); + when(config.getBoolean(META_SYNC_INCREMENTAL)).thenReturn(true); + when(config.getSplitStrings(META_SYNC_PARTITION_FIELDS)).thenReturn(Collections.singletonList("p")); + HiveSyncTool tool = mock(HiveSyncTool.class, CALLS_REAL_METHODS); + tool.config = config; + tool.syncClient = syncClient; + // The mock skips the constructor, which is what normally creates the sync metrics the partition scan is timed with. + Field syncMetrics = HiveSyncTool.class.getDeclaredField("syncMetrics"); + syncMetrics.setAccessible(true); + syncMetrics.set(tool, new HoodieHiveSyncStats(false)); + return tool; + } +}
