This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/branch-0.x by this push:
new 1ca8cc7dc55 [HUDI-7207] Sequentially delete complete instant files in
archival to prevent inconsistency during data reads (#10711)
1ca8cc7dc55 is described below
commit 1ca8cc7dc55de0ae7fcaa3b30939abfca152cbb2
Author: majian <[email protected]>
AuthorDate: Thu Feb 22 10:51:48 2024 +0800
[HUDI-7207] Sequentially delete complete instant files in archival to
prevent inconsistency during data reads (#10711)
---
.../java/org/apache/hudi/client/HoodieTimelineArchiver.java | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
index d4abfa82d59..718f8ad2c46 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java
@@ -594,11 +594,13 @@ public class HoodieTimelineArchiver<T extends
HoodieAvroPayload, I, K, O> {
);
}
if (!completedInstants.isEmpty()) {
- context.foreach(
- completedInstants,
- instant -> activeTimeline.deleteInstantFileIfExists(instant),
- Math.min(completedInstants.size(),
config.getArchiveDeleteParallelism())
- );
+ // Due to the concurrency between deleting completed instants and
reading data,
+ // there may be hole in the timeline, which can lead to errors when
reading data.
+ // Therefore, the concurrency of deleting completed instants is
temporarily disabled,
+ // and instants are deleted in ascending order to prevent the occurrence
of such holes.
+ // See HUDI-7207 and #10325.
+ completedInstants.stream()
+ .forEach(instant ->
activeTimeline.deleteInstantFileIfExists(instant));
}
return true;
