This is an automated email from the ASF dual-hosted git repository.
sadanand48 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new af3e34377b0 HDDS-16076. Separate test class for
testInstallSnapshotDuringBootstrapping. (#10931)
af3e34377b0 is described below
commit af3e34377b03156666014c760d51c7a36a503b89
Author: Sadanand Shenoy <[email protected]>
AuthorDate: Mon Aug 10 14:24:05 2026 +0530
HDDS-16076. Separate test class for testInstallSnapshotDuringBootstrapping.
(#10931)
---
.../TestOMInstallSnapshotDuringBootstrapping.java | 207 +++++++++++++++++++++
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 141 +-------------
2 files changed, 209 insertions(+), 139 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMInstallSnapshotDuringBootstrapping.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMInstallSnapshotDuringBootstrapping.java
new file mode 100644
index 00000000000..5d0d1724fef
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMInstallSnapshotDuringBootstrapping.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om;
+
+import static
org.apache.hadoop.ozone.om.TestOzoneManagerHAWithStoppedNodes.createKey;
+import static org.apache.ozone.test.OzoneTestBase.uniqueObjectName;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.utils.RDBSnapshotProvider;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneClientFactory;
+import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
+import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServerConfig;
+import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine;
+import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ozone.test.GenericTestUtils.LogCapturer;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Regression tests for OM bootstrap install snapshot while {@code
BOOTSTRAPPING}.
+ */
+public class TestOMInstallSnapshotDuringBootstrapping {
+
+ private static final String OM_SERVICE_ID = "om-service-bootstrap";
+ private static final int LOG_PURGE_GAP = 5;
+ private static final long SNAPSHOT_THRESHOLD = 50;
+ private static final long TARGET_LOG_INDEX = 200;
+ private static final int INSTALL_START_DEADLINE_MS = 30_000;
+ private static final int COMPLETION_DEADLINE_MS = 60_000;
+ private static final BucketLayout TEST_BUCKET_LAYOUT =
BucketLayout.OBJECT_STORE;
+
+ private MiniOzoneHAClusterImpl cluster;
+ private OzoneClient client;
+ private OzoneBucket ozoneBucket;
+
+ @BeforeEach
+ public void init() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setInt(OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, 5);
+ conf.setInt(OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_GAP, LOG_PURGE_GAP);
+
conf.setLong(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY,
+ SNAPSHOT_THRESHOLD);
+ conf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_KEY, 16,
StorageUnit.KB);
+
conf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
+ 16, StorageUnit.KB);
+
+ OzoneManagerRatisServerConfig omRatisConf =
+ conf.getObject(OzoneManagerRatisServerConfig.class);
+ omRatisConf.setLogAppenderWaitTimeMin(10);
+ conf.setFromObject(omRatisConf);
+
+ cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
+ .setOMServiceId(OM_SERVICE_ID)
+ .setNumOfOzoneManagers(2)
+ .setNumDatanodes(1)
+ .build();
+ cluster.waitForClusterToBeReady();
+
+ client = OzoneClientFactory.getRpcClient(OM_SERVICE_ID, conf);
+ ObjectStore objectStore = client.getObjectStore();
+ String volumeName = uniqueObjectName("volume");
+ String bucketName = uniqueObjectName("bucket");
+ objectStore.createVolume(volumeName);
+ OzoneVolume volume = objectStore.getVolume(volumeName);
+ volume.createBucket(bucketName,
+ BucketArgs.newBuilder().setBucketLayout(TEST_BUCKET_LAYOUT).build());
+ ozoneBucket = volume.getBucket(bucketName);
+ }
+
+ @AfterEach
+ public void shutdown() {
+ IOUtils.closeQuietly(client);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Checkpoint install must proceed during {@code BOOTSTRAPPING} with the
default
+ * v2 checkpoint API and complete successfully.
+ */
+ @Test
+ public void testInstallSnapshotDuringBootstrapping() throws Exception {
+ OzoneManager leader = cluster.getOMLeader();
+ writeKeysToIncreaseLogIndex(leader.getOmRatisServer(), TARGET_LOG_INDEX);
+ assertThat(leader.getRatisSnapshotIndex())
+ .as("leader should have purged early logs")
+ .isGreaterThan((long) LOG_PURGE_GAP);
+
+ LogCapturer omLog = LogCapturer.captureLogs(OzoneManager.class);
+ LogCapturer stateMachineLog =
+ LogCapturer.captureLogs(OzoneManagerStateMachine.class);
+ LogCapturer snapshotProviderLog =
+ LogCapturer.captureLogs(RDBSnapshotProvider.class);
+ String newNodeId = "omNode-bootstrap-ratis-snapshots";
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<?> bootstrapFuture = executor.submit(() -> {
+ try {
+ cluster.bootstrapOzoneManager(newNodeId);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ try {
+ waitForCheckpointInstallToStart(omLog, snapshotProviderLog);
+ bootstrapFuture.get(COMPLETION_DEADLINE_MS, TimeUnit.MILLISECONDS);
+ assertBootstrapOmJoinedRatisGroup(newNodeId);
+ } finally {
+ bootstrapFuture.cancel(true);
+ omLog.stopCapturing();
+ stateMachineLog.stopCapturing();
+ snapshotProviderLog.stopCapturing();
+ executor.shutdownNow();
+ }
+
+ assertThat(stateMachineLog.getOutput())
+ .as("Ratis should notify the bootstrapping OM to install a checkpoint")
+ .contains("Received install snapshot notification from OM leader");
+ assertThat(omLog.getOutput())
+ .as("checkpoint install must not be aborted during BOOTSTRAPPING")
+ .doesNotContain("Abort install snapshot from Leader");
+ assertThat(omLog.getOutput())
+ .as("checkpoint installation should finish")
+ .contains("Install Checkpoint is finished");
+ assertThat(snapshotProviderLog.getOutput())
+ .as("checkpoint download should start after install is accepted")
+ .contains("Prepare to download the snapshot from leader OM");
+ assertThat(snapshotProviderLog.getOutput())
+ .as("checkpoint tarball should be assembled on the bootstrapping OM")
+ .contains("DB snapshot transfer is complete.");
+ }
+
+ private void writeKeysToIncreaseLogIndex(OzoneManagerRatisServer
omRatisServer,
+ long targetLogIndex) throws Exception {
+ long logIndex = omRatisServer.getLastAppliedTermIndex().getIndex();
+ while (logIndex < targetLogIndex) {
+ createKey(ozoneBucket);
+ logIndex = omRatisServer.getLastAppliedTermIndex().getIndex();
+ }
+ }
+
+ private void assertBootstrapOmJoinedRatisGroup(String newNodeId) {
+ OzoneManager newOm = cluster.getOzoneManager(newNodeId);
+ assertNotNull(newOm, "Bootstrapped OM should be registered on the
cluster");
+ for (OzoneManager om : cluster.getOzoneManagersList()) {
+ assertTrue(om.doesPeerExist(newNodeId),
+ "New OM node " + newNodeId + " not present in peer list of OM " +
om.getOMNodeId());
+ assertTrue(om.getOmRatisServer().doesPeerExist(newNodeId),
+ "New OM node " + newNodeId + " not present in Ratis peer list of OM "
+ + om.getOMNodeId());
+ }
+ }
+
+ private void waitForCheckpointInstallToStart(LogCapturer omLog,
+ LogCapturer snapshotProviderLog) throws InterruptedException,
TimeoutException {
+ try {
+ GenericTestUtils.waitFor(() -> {
+ if (omLog.getOutput().contains("Abort install snapshot from Leader")) {
+ fail("Checkpoint install was aborted during BOOTSTRAPPING.");
+ }
+ return snapshotProviderLog.getOutput()
+ .contains("Prepare to download the snapshot from leader OM");
+ }, 200, INSTALL_START_DEADLINE_MS);
+ } catch (TimeoutException e) {
+ fail("Checkpoint download did not start within " +
INSTALL_START_DEADLINE_MS
+ + "ms. OzoneManager log: " + omLog.getOutput()
+ + ", RDBSnapshotProvider log: " + snapshotProviderLog.getOutput());
+ }
+ }
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
index 7be575e7162..27cfa6d10e9 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
@@ -30,7 +30,6 @@
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
import java.io.File;
import java.io.IOException;
@@ -54,14 +53,12 @@
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.utils.FaultInjector;
-import org.apache.hadoop.hdds.utils.RDBSnapshotProvider;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.hdds.utils.db.RDBCheckpointUtils;
import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.ObjectStore;
@@ -78,7 +75,6 @@
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServerConfig;
-import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.GenericTestUtils.LogCapturer;
@@ -96,18 +92,13 @@
* Tests the Ratis snapshots feature in OM. These tests do not depend on the
* checkpoint transfer format and run once with the default (inode-based)
* transfer; tests exercising the transfer path under both formats live in
- * {@link TestOMRatisSnapshotTransfer}.
+ * {@link TestOMRatisSnapshotTransfer}. Bootstrap install snapshot coverage
lives in
+ * {@link TestOMInstallSnapshotDuringBootstrapping}.
*/
public class TestOMRatisSnapshots {
private static final String OM_SERVICE_ID = "om-service-test1";
- private static final String BOOTSTRAP_OM_SERVICE_ID = "om-service-bootstrap";
private static final int NUM_OF_OMS = 3;
- private static final int BOOTSTRAP_LOG_PURGE_GAP = 5;
- private static final long BOOTSTRAP_TARGET_LOG_INDEX = 200;
- private static final int BOOTSTRAP_INSTALL_START_DEADLINE_MS = 30_000;
- private static final int BOOTSTRAP_COMPLETION_DEADLINE_MS = 60_000;
-
private MiniOzoneHAClusterImpl cluster = null;
private ObjectStore objectStore;
private OzoneConfiguration conf;
@@ -742,134 +733,6 @@ public void
testInstallSnapshotFromLeaderFailedDownloadCleanupSucceeds()
followerOM.getOmSnapshotProvider().setInjector(null);
}
- /**
- * Regression test for bootstrap when leader logs are purged: checkpoint
install
- * must proceed during {@code BOOTSTRAPPING} with the default v2 checkpoint
API
- * and complete successfully.
- */
- @Test
- public void testBootstrapInstallSnapshotDuringBootstrapping() throws
Exception {
- IOUtils.closeQuietly(client);
- if (cluster != null) {
- cluster.shutdown();
- }
-
- OzoneConfiguration bootstrapConf = new OzoneConfiguration();
-
bootstrapConf.setInt(OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, 5);
- bootstrapConf.setInt(OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_GAP,
BOOTSTRAP_LOG_PURGE_GAP);
-
bootstrapConf.setLong(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY,
- SNAPSHOT_THRESHOLD);
- bootstrapConf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_KEY,
16,
- StorageUnit.KB);
-
bootstrapConf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
- 16, StorageUnit.KB);
-
- OzoneManagerRatisServerConfig omRatisConf =
- bootstrapConf.getObject(OzoneManagerRatisServerConfig.class);
- omRatisConf.setLogAppenderWaitTimeMin(10);
- bootstrapConf.setFromObject(omRatisConf);
-
- cluster = (MiniOzoneHAClusterImpl)
MiniOzoneCluster.newHABuilder(bootstrapConf)
- .setOMServiceId(BOOTSTRAP_OM_SERVICE_ID)
- .setNumOfOzoneManagers(2)
- .setNumDatanodes(1)
- .build();
- cluster.waitForClusterToBeReady();
-
- client = OzoneClientFactory.getRpcClient(BOOTSTRAP_OM_SERVICE_ID,
bootstrapConf);
- objectStore = client.getObjectStore();
- String bootstrapVolume = uniqueObjectName("volume");
- String bootstrapBucket = uniqueObjectName("bucket");
- objectStore.createVolume(bootstrapVolume);
- OzoneVolume volume = objectStore.getVolume(bootstrapVolume);
- volume.createBucket(bootstrapBucket,
- BucketArgs.newBuilder().setBucketLayout(TEST_BUCKET_LAYOUT).build());
- ozoneBucket = volume.getBucket(bootstrapBucket);
-
- OzoneManager leader = cluster.getOMLeader();
- writeKeysToIncreaseLogIndex(leader.getOmRatisServer(),
BOOTSTRAP_TARGET_LOG_INDEX);
- assertThat(leader.getRatisSnapshotIndex())
- .as("leader should have purged early logs")
- .isGreaterThan((long) BOOTSTRAP_LOG_PURGE_GAP);
-
- LogCapturer omLog = LogCapturer.captureLogs(OzoneManager.class);
- LogCapturer stateMachineLog =
- LogCapturer.captureLogs(OzoneManagerStateMachine.class);
- LogCapturer snapshotProviderLog =
- LogCapturer.captureLogs(RDBSnapshotProvider.class);
- String newNodeId = "omNode-bootstrap-ratis-snapshots";
- ExecutorService executor = Executors.newSingleThreadExecutor();
- Future<?> bootstrapFuture = executor.submit(() -> {
- try {
- cluster.bootstrapOzoneManager(newNodeId);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- });
-
- try {
- waitForBootstrapCheckpointInstallToStart(omLog, snapshotProviderLog);
- bootstrapFuture.get(BOOTSTRAP_COMPLETION_DEADLINE_MS,
TimeUnit.MILLISECONDS);
- assertBootstrapOmJoinedRatisGroup(newNodeId);
- } finally {
- bootstrapFuture.cancel(true);
- omLog.stopCapturing();
- stateMachineLog.stopCapturing();
- snapshotProviderLog.stopCapturing();
- executor.shutdownNow();
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- assertThat(stateMachineLog.getOutput())
- .as("Ratis should notify the bootstrapping OM to install a checkpoint")
- .contains("Received install snapshot notification from OM leader");
- assertThat(omLog.getOutput())
- .as("checkpoint install must not be aborted during BOOTSTRAPPING")
- .doesNotContain("Abort install snapshot from Leader");
- assertThat(omLog.getOutput())
- .as("checkpoint installation should finish")
- .contains("Install Checkpoint is finished");
- assertThat(snapshotProviderLog.getOutput())
- .as("checkpoint download should start after install is accepted")
- .contains("Prepare to download the snapshot from leader OM");
- assertThat(snapshotProviderLog.getOutput())
- .as("checkpoint tarball should be assembled on the bootstrapping OM")
- .contains("DB snapshot transfer is complete.");
- }
-
- private void assertBootstrapOmJoinedRatisGroup(String newNodeId) {
- OzoneManager newOm = cluster.getOzoneManager(newNodeId);
- assertNotNull(newOm, "Bootstrapped OM should be registered on the
cluster");
- for (OzoneManager om : cluster.getOzoneManagersList()) {
- assertTrue(om.doesPeerExist(newNodeId),
- "New OM node " + newNodeId + " not present in peer list of OM " +
om.getOMNodeId());
- assertTrue(om.getOmRatisServer().doesPeerExist(newNodeId),
- "New OM node " + newNodeId + " not present in Ratis peer list of OM "
- + om.getOMNodeId());
- }
- }
-
- private void waitForBootstrapCheckpointInstallToStart(
- LogCapturer omLog,
- LogCapturer snapshotProviderLog)
- throws InterruptedException, TimeoutException {
- try {
- GenericTestUtils.waitFor(() -> {
- if (omLog.getOutput().contains("Abort install snapshot from Leader")) {
- fail("Checkpoint install was aborted during BOOTSTRAPPING.");
- }
- return snapshotProviderLog.getOutput()
- .contains("Prepare to download the snapshot from leader OM");
- }, 200, BOOTSTRAP_INSTALL_START_DEADLINE_MS);
- } catch (TimeoutException e) {
- fail("Checkpoint download did not start within " +
BOOTSTRAP_INSTALL_START_DEADLINE_MS
- + "ms. OzoneManager log: " + omLog.getOutput()
- + ", RDBSnapshotProvider log: " + snapshotProviderLog.getOutput());
- }
- }
-
/**
* Moves all contents from the checkpoint location into the omDbDir.
* This reorganizes the checkpoint structure so that all checkpoint files
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
