smengcl commented on code in PR #10917:
URL: https://github.com/apache/ozone/pull/10917#discussion_r3693445477
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java:
##########
@@ -597,6 +606,111 @@ public void
testInstallSnapshotFromLeaderFailedDownloadCleanupSucceeds()
followerOM.getOmSnapshotProvider().setInjector(null);
}
+ /**
+ * Regression test for bootstrap when leader logs are purged: checkpoint
install
+ * must proceed during {@code BOOTSTRAPPING} with the default v2 checkpoint
API.
+ * Stops once download starts so the test targets the BOOTSTRAPPING guard
only.
+ */
+ @Test
+ public void testBootstrapInstallSnapshotDuringBootstrapping() throws
Exception {
+ IOUtils.closeQuietly(client);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+
+ OzoneConfiguration bootstrapConf = new OzoneConfiguration();
+
bootstrapConf.setInt(OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY, 5);
+ bootstrapConf.setInt(OMConfigKeys.OZONE_OM_RATIS_LOG_PURGE_GAP,
BOOTSTRAP_LOG_PURGE_GAP);
+
bootstrapConf.setLong(OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY,
+ SNAPSHOT_THRESHOLD);
+ bootstrapConf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_KEY,
16,
+ StorageUnit.KB);
+
bootstrapConf.setStorageSize(OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
+ 16, StorageUnit.KB);
+
+ OzoneManagerRatisServerConfig omRatisConf =
+ bootstrapConf.getObject(OzoneManagerRatisServerConfig.class);
+ omRatisConf.setLogAppenderWaitTimeMin(10);
+ bootstrapConf.setFromObject(omRatisConf);
+
+ cluster = (MiniOzoneHAClusterImpl)
MiniOzoneCluster.newHABuilder(bootstrapConf)
+ .setOMServiceId(BOOTSTRAP_OM_SERVICE_ID)
+ .setNumOfOzoneManagers(2)
+ .setNumDatanodes(1)
+ .build();
+ cluster.waitForClusterToBeReady();
+
+ client = OzoneClientFactory.getRpcClient(BOOTSTRAP_OM_SERVICE_ID,
bootstrapConf);
+ objectStore = client.getObjectStore();
+ String bootstrapVolume = uniqueObjectName("volume");
+ String bootstrapBucket = uniqueObjectName("bucket");
+ objectStore.createVolume(bootstrapVolume);
+ OzoneVolume volume = objectStore.getVolume(bootstrapVolume);
+ volume.createBucket(bootstrapBucket,
+ BucketArgs.newBuilder().setBucketLayout(TEST_BUCKET_LAYOUT).build());
+ ozoneBucket = volume.getBucket(bootstrapBucket);
+
+ OzoneManager leader = cluster.getOMLeader();
+ writeKeysToIncreaseLogIndex(leader.getOmRatisServer(),
BOOTSTRAP_TARGET_LOG_INDEX);
+ assertThat(leader.getRatisSnapshotIndex())
+ .as("leader should have purged early logs")
+ .isGreaterThan((long) BOOTSTRAP_LOG_PURGE_GAP);
+
+ LogCapturer omLog = LogCapturer.captureLogs(OzoneManager.class);
+ LogCapturer stateMachineLog =
+ LogCapturer.captureLogs(OzoneManagerStateMachine.class);
+ LogCapturer snapshotProviderLog =
+ LogCapturer.captureLogs(RDBSnapshotProvider.class);
+ String newNodeId = "omNode-bootstrap-ratis-snapshots";
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ Future<?> bootstrapFuture = executor.submit(() -> {
+ try {
+ cluster.bootstrapOzoneManager(newNodeId);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ try {
+ waitForBootstrapCheckpointInstallToStart(omLog, snapshotProviderLog);
Review Comment:
Shall we also wait for `bootstrapFuture` to complete and assert that the new
OM joined the Ratis group? Right now the test cancels bootstrap after download
starts and can pass when bootstrap fails.
```diff
@@
private static final int BOOTSTRAP_INSTALL_START_DEADLINE_MS = 30_000;
+ private static final int BOOTSTRAP_COMPLETION_DEADLINE_MS = 60_000;
@@
try {
waitForBootstrapCheckpointInstallToStart(omLog, snapshotProviderLog);
+ bootstrapFuture.get(
+ BOOTSTRAP_COMPLETION_DEADLINE_MS, TimeUnit.MILLISECONDS);
} finally {
bootstrapFuture.cancel(true);
executor.shutdownNow();
@@
assertThat(omLog.getOutput())
.as("checkpoint install must not be aborted during BOOTSTRAPPING")
.doesNotContain("Abort install snapshot from Leader");
+ assertThat(omLog.getOutput())
+ .as("checkpoint installation should finish")
+ .contains("Install Checkpoint is finished");
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
