This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 527b0ec81308 fix(common): load single archived instant details (#19385)
527b0ec81308 is described below
commit 527b0ec813084886edd3e52674af854cfc5ca621
Author: Danny Chan <[email protected]>
AuthorDate: Tue Jul 28 18:04:06 2026 +0800
fix(common): load single archived instant details (#19385)
* Fix loading single archived compaction plan
* Fix completed instant detail range
---
.../table/timeline/TestArchivedTimelineV2.java | 69 ++++++++++++++++++++--
.../timeline/versioning/v2/ArchivedTimelineV2.java | 4 +-
2 files changed, 65 insertions(+), 8 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV2.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV2.java
index 6b10b778c73b..4444fa617cf5 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV2.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/table/timeline/TestArchivedTimelineV2.java
@@ -26,8 +26,10 @@ import
org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.engine.LocalTaskContextSupplier;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndex;
@@ -38,6 +40,7 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import static
org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_GENERATOR;
@@ -46,6 +49,8 @@ import static
org.apache.hudi.common.testutils.HoodieTestUtils.convertMetadataTo
import static
org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
/**
@@ -77,6 +82,54 @@ public class TestArchivedTimelineV2 extends
HoodieCommonTestHarness {
assertThat(archivedTimeline.firstInstant().map(HoodieInstant::requestedTime).orElse(""),
is("10000011"));
}
+ @Test
+ void testLoadCompactionDetailsForSingleInstant() {
+ String instantTime = "10000001";
+ byte[] compactionPlan = {1, 2, 3};
+ HoodieInstant completed = INSTANT_GENERATOR.createNewInstant(
+ HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION,
instantTime, "10000002");
+ ActiveAction activeAction = new DummyActiveAction(completed, new byte[0]) {
+ @Override
+ public String getPendingAction() {
+ return HoodieTimeline.COMPACTION_ACTION;
+ }
+
+ @Override
+ public Option<byte[]> getCompactionPlan(HoodieTableMetaClient
metaClient) {
+ return Option.of(compactionPlan);
+ }
+ };
+ createTimelineWriter().write(
+ Collections.singletonList(activeAction), Option.empty(),
Option.empty());
+
+ HoodieArchivedTimeline archivedTimeline = metaClient.getArchivedTimeline();
+ HoodieInstant archivedInstant = archivedTimeline.firstInstant().get();
+
assertFalse(archivedTimeline.getInstantDetails(archivedInstant).isPresent());
+
+ archivedTimeline.loadCompactionDetailsInMemory(instantTime);
+
+ assertArrayEquals(compactionPlan,
archivedTimeline.getInstantDetails(archivedInstant).get());
+ }
+
+ @Test
+ void testLoadCompletedDetailsForSingleInstant() {
+ String instantTime = "10000001";
+ byte[] commitMetadata = {1, 2, 3};
+ HoodieInstant completed = INSTANT_GENERATOR.createNewInstant(
+ HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION,
instantTime, "10000002");
+ createTimelineWriter().write(
+ Collections.singletonList(new DummyActiveAction(completed,
commitMetadata)),
+ Option.empty(), Option.empty());
+
+ HoodieArchivedTimeline archivedTimeline = metaClient.getArchivedTimeline();
+ HoodieInstant archivedInstant = archivedTimeline.firstInstant().get();
+
assertFalse(archivedTimeline.getInstantDetails(archivedInstant).isPresent());
+
+ archivedTimeline.loadCompletedInstantDetailsInMemory(instantTime,
instantTime);
+
+ assertArrayEquals(commitMetadata,
archivedTimeline.getInstantDetails(archivedInstant).get());
+ }
+
@Test
void getInstantReaderReferencesSelf() {
HoodieArchivedTimeline timeline =
TIMELINE_FACTORY.createArchivedTimeline(metaClient);
@@ -88,12 +141,8 @@ public class TestArchivedTimelineV2 extends
HoodieCommonTestHarness {
private void writeArchivedTimeline(int batchSize, long startTs) throws
Exception {
HoodieTestTable testTable = HoodieTestTable.of(this.metaClient);
- HoodieWriteConfig writeConfig =
HoodieWriteConfig.newBuilder().withPath(this.metaClient.getBasePath())
-
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
- .withMarkersType("DIRECT")
- .build();
+ LSMTimelineWriter writer = createTimelineWriter();
HoodieEngineContext engineContext = new
HoodieLocalEngineContext(getDefaultStorageConf());
- LSMTimelineWriter writer = LSMTimelineWriter.getInstance(writeConfig, new
LocalTaskContextSupplier(), metaClient);
List<ActiveAction> instantBuffer = new ArrayList<>();
for (int i = 1; i <= 50; i++) {
long instantTimeTs = startTs + i;
@@ -105,10 +154,18 @@ public class TestArchivedTimelineV2 extends
HoodieCommonTestHarness {
instantBuffer.add(new DummyActiveAction(instant, serializedMetadata));
if (i % batchSize == 0) {
// archive 10 instants each time
- writer.write(instantBuffer,
org.apache.hudi.common.util.Option.empty(),
org.apache.hudi.common.util.Option.empty());
+ writer.write(instantBuffer, Option.empty(), Option.empty());
writer.compactAndClean(engineContext);
instantBuffer.clear();
}
}
}
+
+ private LSMTimelineWriter createTimelineWriter() {
+ HoodieWriteConfig writeConfig =
HoodieWriteConfig.newBuilder().withPath(this.metaClient.getBasePath())
+
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
+ .withMarkersType("DIRECT")
+ .build();
+ return LSMTimelineWriter.getInstance(writeConfig, new
LocalTaskContextSupplier(), metaClient);
+ }
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java
index 9a181cf9bcbe..f88168ea985c 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/versioning/v2/ArchivedTimelineV2.java
@@ -147,7 +147,7 @@ public class ArchivedTimelineV2 extends BaseTimelineV2
implements HoodieArchived
public void loadCompactionDetailsInMemory(String startTs, String endTs) {
// load compactionPlan
- List<HoodieInstant> loadedInstants = loadInstants(new
HoodieArchivedTimeline.TimeRangeFilter(startTs, endTs),
HoodieArchivedTimeline.LoadMode.PLAN,
+ List<HoodieInstant> loadedInstants = loadInstants(new
HoodieArchivedTimeline.ClosedClosedTimeRangeFilter(startTs, endTs),
HoodieArchivedTimeline.LoadMode.PLAN,
record ->
record.get(ACTION_ARCHIVED_META_FIELD).toString().equals(COMMIT_ACTION)
&& record.get(PLAN_ARCHIVED_META_FIELD) != null
);
@@ -164,7 +164,7 @@ public class ArchivedTimelineV2 extends BaseTimelineV2
implements HoodieArchived
@Override
public void loadCompletedInstantDetailsInMemory(String startTs, String
endTs) {
- List<HoodieInstant> loadedInstants = loadInstants(new
HoodieArchivedTimeline.TimeRangeFilter(startTs, endTs),
HoodieArchivedTimeline.LoadMode.METADATA);
+ List<HoodieInstant> loadedInstants = loadInstants(new
HoodieArchivedTimeline.ClosedClosedTimeRangeFilter(startTs, endTs),
HoodieArchivedTimeline.LoadMode.METADATA);
appendLoadedInstants(loadedInstants);
}
