This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new fcf5b17a4e HDDS-8665. Implemented CopyObject for SnapshotInfo and
moved out snapshot chain update form OMSnapshotPurgeResponse (#5201)
fcf5b17a4e is described below
commit fcf5b17a4ede0c55c76aa337438498361c0a5dd3
Author: Hemant Kumar <[email protected]>
AuthorDate: Wed Aug 23 10:31:25 2023 -0700
HDDS-8665. Implemented CopyObject for SnapshotInfo and moved out snapshot
chain update form OMSnapshotPurgeResponse (#5201)
---
.../hadoop/ozone/om/helpers/SnapshotInfo.java | 32 ++++-
.../hadoop/ozone/om/SnapshotChainManager.java | 12 ++
.../request/snapshot/OMSnapshotPurgeRequest.java | 101 ++++++++++++++-
.../response/snapshot/OMSnapshotPurgeResponse.java | 101 +++------------
.../TestOMSnapshotPurgeRequestAndResponse.java | 137 +++++++++++++--------
5 files changed, 239 insertions(+), 144 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
index fcd28ce329..bdb642d8fb 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.ozone.om.helpers;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.utils.db.Codec;
+import org.apache.hadoop.hdds.utils.db.CopyObject;
import org.apache.hadoop.hdds.utils.db.DelegatedCodec;
import org.apache.hadoop.hdds.utils.db.Proto2Codec;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -52,15 +53,11 @@ import static
org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
* snapshot checkpoint directory, previous snapshotid
* for the snapshot path & global amongst other necessary fields.
*/
-public final class SnapshotInfo implements Auditable {
+public final class SnapshotInfo implements Auditable, CopyObject<SnapshotInfo>
{
private static final Codec<SnapshotInfo> CODEC = new DelegatedCodec<>(
Proto2Codec.get(OzoneManagerProtocolProtos.SnapshotInfo.class),
SnapshotInfo::getFromProtobuf,
- SnapshotInfo::getProtobuf,
- // FIXME: HDDS-8665 Deep copy will cause failures
- // - TestOMSnapshotDeleteRequest NullPointerException
- // - TestOMSnapshotPurgeRequestAndResponse AssertionFailedError
- DelegatedCodec.CopyType.SHALLOW);
+ SnapshotInfo::getProtobuf);
public static Codec<SnapshotInfo> getCodec() {
return CODEC;
@@ -599,4 +596,27 @@ public final class SnapshotInfo implements Auditable {
creationTime, deletionTime, pathPreviousSnapshotId,
globalPreviousSnapshotId, snapshotPath, checkpointDir);
}
+
+ /**
+ * Return a new copy of the object.
+ */
+ @Override
+ public SnapshotInfo copyObject() {
+ return new Builder()
+ .setSnapshotId(snapshotId)
+ .setName(name)
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setSnapshotStatus(snapshotStatus)
+ .setCreationTime(creationTime)
+ .setDeletionTime(deletionTime)
+ .setPathPreviousSnapshotId(pathPreviousSnapshotId)
+ .setGlobalPreviousSnapshotId(globalPreviousSnapshotId)
+ .setSnapshotPath(snapshotPath)
+ .setCheckpointDir(checkpointDir)
+ .setDbTxSequenceNumber(dbTxSequenceNumber)
+ .setDeepClean(deepClean)
+ .setSstFiltered(sstFiltered)
+ .build();
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java
index e4f66b6512..b47d851519 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SnapshotChainManager.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.ozone.om;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
@@ -430,4 +431,15 @@ public class SnapshotChainManager {
String path) {
return snapshotChainByPath.get(path);
}
+
+ @VisibleForTesting
+ public Map<UUID, SnapshotChainInfo> getGlobalSnapshotChain() {
+ return globalSnapshotChain;
+ }
+
+ @VisibleForTesting
+ public Map<String,
+ LinkedHashMap<UUID, SnapshotChainInfo>> getSnapshotChainByPath() {
+ return snapshotChainByPath;
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java
index 414ec0f67b..43943d353d 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotPurgeRequest.java
@@ -40,6 +40,8 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.UUID;
/**
* Handles OMSnapshotPurge Request.
@@ -73,6 +75,8 @@ public class OMSnapshotPurgeRequest extends OMClientRequest {
List<String> snapInfosToUpdate = snapshotPurgeRequest
.getUpdatedSnapshotDBKeyList();
Map<String, SnapshotInfo> updatedSnapInfos = new HashMap<>();
+ Map<String, SnapshotInfo> updatedPathPreviousAndGlobalSnapshots =
+ new HashMap<>();
// Snapshots that are already deepCleaned by the KeyDeletingService
// can be marked as deepCleaned.
@@ -94,12 +98,16 @@ public class OMSnapshotPurgeRequest extends OMClientRequest
{
SnapshotInfo nextSnapshot = SnapshotUtils
.getNextActiveSnapshot(fromSnapshot,
snapshotChainManager, omSnapshotManager);
+
updateSnapshotInfoAndCache(nextSnapshot, omMetadataManager,
trxnLogIndex, updatedSnapInfos, true);
+ updateSnapshotChainAndCache(omMetadataManager, fromSnapshot,
+ trxnLogIndex, updatedPathPreviousAndGlobalSnapshots);
}
omClientResponse = new OMSnapshotPurgeResponse(omResponse.build(),
- snapshotDbKeys, updatedSnapInfos);
+ snapshotDbKeys, updatedSnapInfos,
+ updatedPathPreviousAndGlobalSnapshots);
} catch (IOException ex) {
omClientResponse = new OMSnapshotPurgeResponse(
createErrorOMResponse(omResponse, ex));
@@ -124,4 +132,95 @@ public class OMSnapshotPurgeRequest extends
OMClientRequest {
updatedSnapInfos.put(snapInfo.getTableKey(), snapInfo);
}
}
+
+ /**
+ * Removes the snapshot from the chain and updates the next snapshot's
+ * previousPath and previousGlobal IDs in DB cache.
+ * It also returns the pair of updated next path and global snapshots to
+ * update in DB.
+ */
+ private void updateSnapshotChainAndCache(
+ OmMetadataManagerImpl metadataManager,
+ SnapshotInfo snapInfo,
+ long trxnLogIndex,
+ Map<String, SnapshotInfo> updatedPathPreviousAndGlobalSnapshots
+ ) throws IOException {
+ if (snapInfo == null) {
+ return;
+ }
+
+ SnapshotChainManager snapshotChainManager = metadataManager
+ .getSnapshotChainManager();
+ SnapshotInfo nextPathSnapInfo = null;
+
+ // If the snapshot is deleted in the previous run, then the in-memory
+ // SnapshotChainManager might throw NoSuchElementException as the snapshot
+ // is removed in-memory but OMDoubleBuffer has not flushed yet.
+ boolean hasNextPathSnapshot;
+ boolean hasNextGlobalSnapshot;
+ try {
+ hasNextPathSnapshot = snapshotChainManager.hasNextPathSnapshot(
+ snapInfo.getSnapshotPath(), snapInfo.getSnapshotId());
+ hasNextGlobalSnapshot = snapshotChainManager.hasNextGlobalSnapshot(
+ snapInfo.getSnapshotId());
+ } catch (NoSuchElementException ex) {
+ return;
+ }
+
+ // Updates next path snapshot's previous snapshot ID
+ if (hasNextPathSnapshot) {
+ UUID nextPathSnapshotId = snapshotChainManager.nextPathSnapshot(
+ snapInfo.getSnapshotPath(), snapInfo.getSnapshotId());
+
+ String snapshotTableKey = snapshotChainManager
+ .getTableKey(nextPathSnapshotId);
+ nextPathSnapInfo = metadataManager.getSnapshotInfoTable()
+ .get(snapshotTableKey);
+ if (nextPathSnapInfo != null) {
+ nextPathSnapInfo.setPathPreviousSnapshotId(
+ snapInfo.getPathPreviousSnapshotId());
+ metadataManager.getSnapshotInfoTable().addCacheEntry(
+ new CacheKey<>(nextPathSnapInfo.getTableKey()),
+ CacheValue.get(trxnLogIndex, nextPathSnapInfo));
+ updatedPathPreviousAndGlobalSnapshots
+ .put(nextPathSnapInfo.getTableKey(), nextPathSnapInfo);
+ }
+ }
+
+ // Updates next global snapshot's previous snapshot ID
+ if (hasNextGlobalSnapshot) {
+ UUID nextGlobalSnapshotId =
+ snapshotChainManager.nextGlobalSnapshot(snapInfo.getSnapshotId());
+
+ String snapshotTableKey = snapshotChainManager
+ .getTableKey(nextGlobalSnapshotId);
+
+ SnapshotInfo nextGlobalSnapInfo = metadataManager.getSnapshotInfoTable()
+ .get(snapshotTableKey);
+ // If both next global and path snapshot are same, it may overwrite
+ // nextPathSnapInfo.setPathPreviousSnapshotID(), adding this check
+ // will prevent it.
+ if (nextGlobalSnapInfo != null && nextPathSnapInfo != null &&
+ nextGlobalSnapInfo.getSnapshotId().equals(
+ nextPathSnapInfo.getSnapshotId())) {
+ nextPathSnapInfo.setGlobalPreviousSnapshotId(
+ snapInfo.getGlobalPreviousSnapshotId());
+ metadataManager.getSnapshotInfoTable().addCacheEntry(
+ new CacheKey<>(nextPathSnapInfo.getTableKey()),
+ CacheValue.get(trxnLogIndex, nextPathSnapInfo));
+ updatedPathPreviousAndGlobalSnapshots
+ .put(nextPathSnapInfo.getTableKey(), nextPathSnapInfo);
+ } else if (nextGlobalSnapInfo != null) {
+ nextGlobalSnapInfo.setGlobalPreviousSnapshotId(
+ snapInfo.getGlobalPreviousSnapshotId());
+ metadataManager.getSnapshotInfoTable().addCacheEntry(
+ new CacheKey<>(nextGlobalSnapInfo.getTableKey()),
+ CacheValue.get(trxnLogIndex, nextGlobalSnapInfo));
+ updatedPathPreviousAndGlobalSnapshots
+ .put(nextGlobalSnapInfo.getTableKey(), nextGlobalSnapInfo);
+ }
+ }
+
+ snapshotChainManager.deleteSnapshot(snapInfo);
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
index abc68a4cfc..863bf5f62b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/snapshot/OMSnapshotPurgeResponse.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
-import org.apache.hadoop.ozone.om.SnapshotChainManager;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
@@ -37,8 +36,6 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.UUID;
import static
org.apache.hadoop.ozone.om.OmMetadataManagerImpl.SNAPSHOT_INFO_TABLE;
@@ -51,13 +48,18 @@ public class OMSnapshotPurgeResponse extends
OMClientResponse {
LoggerFactory.getLogger(OMSnapshotPurgeResponse.class);
private final List<String> snapshotDbKeys;
private final Map<String, SnapshotInfo> updatedSnapInfos;
+ private final Map<String, SnapshotInfo> updatedPreviousAndGlobalSnapInfos;
- public OMSnapshotPurgeResponse(@Nonnull OMResponse omResponse,
+ public OMSnapshotPurgeResponse(
+ @Nonnull OMResponse omResponse,
@Nonnull List<String> snapshotDbKeys,
- Map<String, SnapshotInfo> updatedSnapInfos) {
+ Map<String, SnapshotInfo> updatedSnapInfos,
+ Map<String, SnapshotInfo> updatedPreviousAndGlobalSnapInfos
+ ) {
super(omResponse);
this.snapshotDbKeys = snapshotDbKeys;
this.updatedSnapInfos = updatedSnapInfos;
+ this.updatedPreviousAndGlobalSnapInfos = updatedPreviousAndGlobalSnapInfos;
}
/**
@@ -69,6 +71,7 @@ public class OMSnapshotPurgeResponse extends OMClientResponse
{
checkStatusNotOK();
this.snapshotDbKeys = null;
this.updatedSnapInfos = null;
+ this.updatedPreviousAndGlobalSnapInfos = null;
}
@Override
@@ -77,7 +80,9 @@ public class OMSnapshotPurgeResponse extends OMClientResponse
{
OmMetadataManagerImpl metadataManager = (OmMetadataManagerImpl)
omMetadataManager;
- updateSnapInfo(metadataManager, batchOperation);
+ updateSnapInfo(metadataManager, batchOperation, updatedSnapInfos);
+ updateSnapInfo(metadataManager, batchOperation,
+ updatedPreviousAndGlobalSnapInfos);
for (String dbKey: snapshotDbKeys) {
SnapshotInfo snapshotInfo = omMetadataManager
.getSnapshotInfoTable().get(dbKey);
@@ -88,7 +93,7 @@ public class OMSnapshotPurgeResponse extends OMClientResponse
{
if (snapshotInfo == null) {
continue;
}
- cleanupSnapshotChain(metadataManager, snapshotInfo, batchOperation);
+
// Delete Snapshot checkpoint directory.
deleteCheckpointDirectory(omMetadataManager, snapshotInfo);
omMetadataManager.getSnapshotInfoTable().deleteWithBatch(batchOperation,
@@ -97,91 +102,15 @@ public class OMSnapshotPurgeResponse extends
OMClientResponse {
}
private void updateSnapInfo(OmMetadataManagerImpl metadataManager,
- BatchOperation batchOp)
+ BatchOperation batchOp,
+ Map<String, SnapshotInfo> snapshotInfos)
throws IOException {
- for (Map.Entry<String, SnapshotInfo> entry : updatedSnapInfos.entrySet()) {
+ for (Map.Entry<String, SnapshotInfo> entry : snapshotInfos.entrySet()) {
metadataManager.getSnapshotInfoTable().putWithBatch(batchOp,
entry.getKey(), entry.getValue());
}
}
- /**
- * Cleans up the snapshot chain and updates next snapshot's
- * previousPath and previousGlobal IDs.
- */
- private void cleanupSnapshotChain(OmMetadataManagerImpl metadataManager,
- SnapshotInfo snapInfo,
- BatchOperation batchOperation)
- throws IOException {
- SnapshotChainManager snapshotChainManager = metadataManager
- .getSnapshotChainManager();
- SnapshotInfo nextPathSnapInfo = null;
- SnapshotInfo nextGlobalSnapInfo;
-
- // If the snapshot is deleted in the previous run, then the in-memory
- // SnapshotChainManager might throw NoSuchElementException as the snapshot
- // is removed in-memory but OMDoubleBuffer has not flushed yet.
- boolean hasNextPathSnapshot;
- boolean hasNextGlobalSnapshot;
- try {
- hasNextPathSnapshot = snapshotChainManager.hasNextPathSnapshot(
- snapInfo.getSnapshotPath(), snapInfo.getSnapshotId());
- hasNextGlobalSnapshot = snapshotChainManager.hasNextGlobalSnapshot(
- snapInfo.getSnapshotId());
- } catch (NoSuchElementException ex) {
- LOG.warn("The Snapshot {} could have been deleted in the previous run.",
- snapInfo.getSnapshotId(), ex);
- return;
- }
-
- // Updates next path snapshot's previous snapshot ID
- if (hasNextPathSnapshot) {
- UUID nextPathSnapshotId = snapshotChainManager.nextPathSnapshot(
- snapInfo.getSnapshotPath(), snapInfo.getSnapshotId());
-
- String snapshotTableKey = snapshotChainManager
- .getTableKey(nextPathSnapshotId);
- nextPathSnapInfo = metadataManager.getSnapshotInfoTable()
- .get(snapshotTableKey);
- if (nextPathSnapInfo != null) {
- nextPathSnapInfo.setPathPreviousSnapshotId(
- snapInfo.getPathPreviousSnapshotId());
- metadataManager.getSnapshotInfoTable().putWithBatch(batchOperation,
- nextPathSnapInfo.getTableKey(), nextPathSnapInfo);
- }
- }
-
- // Updates next global snapshot's previous snapshot ID
- if (hasNextGlobalSnapshot) {
- UUID nextGlobalSnapshotId =
- snapshotChainManager.nextGlobalSnapshot(snapInfo.getSnapshotId());
-
- String snapshotTableKey = snapshotChainManager
- .getTableKey(nextGlobalSnapshotId);
- nextGlobalSnapInfo = metadataManager.getSnapshotInfoTable()
- .get(snapshotTableKey);
- // If both next global and path snapshot are same, it may overwrite
- // nextPathSnapInfo.setPathPreviousSnapshotID(), adding this check
- // will prevent it.
- if (nextGlobalSnapInfo != null && nextPathSnapInfo != null &&
- nextGlobalSnapInfo.getSnapshotId().equals(
- nextPathSnapInfo.getSnapshotId())) {
- nextPathSnapInfo.setGlobalPreviousSnapshotId(
- snapInfo.getGlobalPreviousSnapshotId());
- metadataManager.getSnapshotInfoTable().putWithBatch(batchOperation,
- nextPathSnapInfo.getTableKey(), nextPathSnapInfo);
- } else if (nextGlobalSnapInfo != null) {
- nextGlobalSnapInfo.setGlobalPreviousSnapshotId(
- snapInfo.getGlobalPreviousSnapshotId());
- metadataManager.getSnapshotInfoTable().putWithBatch(batchOperation,
- nextGlobalSnapInfo.getTableKey(), nextGlobalSnapInfo);
- }
- }
-
- // Removes current snapshot from the snapshot chain.
- snapshotChainManager.deleteSnapshot(snapInfo);
- }
-
/**
* Deletes the checkpoint directory for a snapshot.
*/
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java
index fc54bc0d89..06a30e1469 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/snapshot/TestOMSnapshotPurgeRequestAndResponse.java
@@ -58,7 +58,6 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -259,6 +258,8 @@ public class TestOMSnapshotPurgeRequestAndResponse {
}
}
+ // TODO: clean up: Do we this test after
+ // testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge?
@ParameterizedTest
@ValueSource(ints = {0, 1, 2, 3, 4})
public void testSnapshotChainCleanup(int index) throws Exception {
@@ -331,49 +332,72 @@ public class TestOMSnapshotPurgeRequestAndResponse {
private static Stream<Arguments> snapshotPurgeCases() {
return Stream.of(
- Arguments.of(0, true),
- Arguments.of(1, true),
- Arguments.of(2, true),
- Arguments.of(3, true),
- Arguments.of(4, true),
- Arguments.of(5, true),
- Arguments.of(6, true),
- Arguments.of(7, true),
- Arguments.of(8, true),
- Arguments.of(0, false),
- Arguments.of(1, false),
- Arguments.of(2, false),
- Arguments.of(3, false),
- Arguments.of(4, false),
- Arguments.of(5, false),
- Arguments.of(6, false),
- Arguments.of(7, false),
- Arguments.of(8, false)
+ Arguments.of("Single bucket: purge first snapshot.",
+ 1, 5, 0, 0, true),
+ Arguments.of("Single bucket: purge snapshot at index 2.",
+ 1, 5, 2, 2, true),
+ Arguments.of("Single bucket: purge snapshots from index 1 to 3.",
+ 1, 5, 1, 3, true),
+ Arguments.of("Single bucket: purge last snapshot.",
+ 1, 5, 4, 4, true),
+ Arguments.of("Multiple buckets (keys are created in bucket order): " +
+ "purge first snapshot.", 3, 5, 0, 0, true),
+ Arguments.of("Multiple buckets (keys are created in bucket order): " +
+ "purge first 5 snapshots.", 3, 5, 0, 4, true),
+ Arguments.of("Multiple buckets (keys are created in bucket order): " +
+ "purge snapshot at index 7.", 3, 5, 7, 7, true),
+ Arguments.of("Multiple buckets (keys are created in bucket order): " +
+ "purge snapshots from index 5 to 9.", 3, 5, 5, 9, true),
+ Arguments.of("Multiple buckets (keys are created in bucket order): " +
+ "purge snapshots from index 3 to 12.", 3, 5, 3, 12, true),
+ Arguments.of("Multiple buckets (keys are created in bucket order): " +
+ "purge last 5 snapshots.", 3, 5, 10, 14, true),
+ Arguments.of("Multiple buckets (keys are created in bucket order): " +
+ "purge last snapshot.", 3, 5, 14, 14, true),
+ Arguments.of("Multiple buckets (keys are not created in bucket " +
+ "order): purge first snapshot.", 3, 5, 0, 0, false),
+ Arguments.of("Multiple buckets (keys are not created in bucket " +
+ "order): purge first 5 snapshots.", 3, 5, 0, 5, false),
+ Arguments.of("Multiple buckets (keys are not created in bucket " +
+ "order): purge snapshot at index 7.", 3, 5, 7, 7, false),
+ Arguments.of("Multiple buckets (keys are not created in bucket " +
+ "order): purge snapshots from index 5 to 9.", 3, 5, 5, 9, false),
+ Arguments.of("Multiple buckets (keys are not created in bucket " +
+ "order): purge snapshots from index 3 to 12.", 3, 5, 3, 12, false),
+ Arguments.of("Multiple buckets (keys are not created in bucket " +
+ "order): purge last 5 snapshots.", 3, 5, 10, 14, false),
+ Arguments.of("Multiple buckets (keys are not created in bucket " +
+ "order): purge last snapshot.", 3, 5, 14, 14, false)
);
}
- @ParameterizedTest
+ @ParameterizedTest(name = "{0}")
@MethodSource("snapshotPurgeCases")
public void testSnapshotChainInSnapshotInfoTableAfterSnapshotPurge(
- int purgeIndex,
+ String description,
+ int numberOfBuckets,
+ int numberOfKeysPerBucket,
+ int fromIndex,
+ int toIndex,
boolean createInBucketOrder) throws Exception {
- List<String> buckets = Arrays.asList(
- "buck-1-" + UUID.randomUUID(),
- "buck-2-" + UUID.randomUUID(),
- "buck-3-" + UUID.randomUUID()
- );
-
- for (String bucket : buckets) {
- OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucket,
+ SnapshotChainManager chainManager =
+ ((OmMetadataManagerImpl) omMetadataManager).getSnapshotChainManager();
+ int totalKeys = numberOfBuckets * numberOfKeysPerBucket;
+
+ List<String> buckets = new ArrayList<>();
+ for (int i = 0; i < numberOfBuckets; i++) {
+ String bucketNameLocal = "bucket-" + UUID.randomUUID();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketNameLocal,
omMetadataManager);
+ buckets.add(bucketNameLocal);
}
List<SnapshotInfo> snapshotInfoList = new ArrayList<>();
- for (int i = 0; i < 3; i++) {
- for (int j = 0; j < 3; j++) {
+ for (int i = 0; i < numberOfBuckets; i++) {
+ for (int j = 0; j < numberOfKeysPerBucket; j++) {
int bucketIndex = createInBucketOrder ? i : j;
- String bucket = buckets.get(bucketIndex % 3);
+ String bucket = buckets.get(bucketIndex % numberOfBuckets);
String snapshotName = UUID.randomUUID().toString();
createSnapshotCheckpoint(volumeName, bucket, snapshotName);
String snapshotTableKey =
@@ -384,33 +408,44 @@ public class TestOMSnapshotPurgeRequestAndResponse {
}
}
- validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain(snapshotInfoList);
+ long numberOfSnapshotBeforePurge = omMetadataManager
+ .countRowsInTable(omMetadataManager.getSnapshotInfoTable());
+ assertEquals(totalKeys, numberOfSnapshotBeforePurge);
+ assertEquals(totalKeys, chainManager.getGlobalSnapshotChain().size());
- SnapshotInfo purgeSnapshotInfo = snapshotInfoList.get(purgeIndex);
+ validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain(snapshotInfoList);
- String purgeSnapshotKey = SnapshotInfo.getTableKey(volumeName,
- purgeSnapshotInfo.getBucketName(),
- purgeSnapshotInfo.getName());
+ List<String> purgeSnapshotKeys = new ArrayList<>();
+ for (int i = fromIndex; i <= toIndex; i++) {
+ SnapshotInfo purgeSnapshotInfo = snapshotInfoList.get(i);
+ String purgeSnapshotKey = SnapshotInfo.getTableKey(volumeName,
+ purgeSnapshotInfo.getBucketName(),
+ purgeSnapshotInfo.getName());
+ purgeSnapshotKeys.add(purgeSnapshotKey);
+ }
- OMRequest snapshotPurgeRequest = createPurgeKeysRequest(
- Collections.singletonList(purgeSnapshotKey));
+ OMRequest snapshotPurgeRequest = createPurgeKeysRequest(purgeSnapshotKeys);
purgeSnapshots(snapshotPurgeRequest);
List<SnapshotInfo> snapshotInfoListAfterPurge = new ArrayList<>();
- for (int i = 0; i < 9; i++) {
- if (i == purgeIndex) {
- // Ignoring purgeIndex because snapshot at purgeIndex has been purged.
- continue;
+ for (int i = 0; i < totalKeys; i++) {
+ if (i < fromIndex || i > toIndex) {
+ SnapshotInfo info = snapshotInfoList.get(i);
+ String snapshotKey = SnapshotInfo.getTableKey(volumeName,
+ info.getBucketName(), info.getName());
+ snapshotInfoListAfterPurge.add(
+ omMetadataManager.getSnapshotInfoTable().get(snapshotKey));
}
-
- SnapshotInfo info = snapshotInfoList.get(i);
- String snapshotKey = SnapshotInfo.getTableKey(volumeName,
- info.getBucketName(),
- info.getName());
-
- snapshotInfoListAfterPurge.add(
- omMetadataManager.getSnapshotInfoTable().get(snapshotKey));
}
+
+ long expectNumberOfSnapshotAfterPurge = totalKeys -
+ (toIndex - fromIndex + 1);
+ long actualNumberOfSnapshotAfterPurge = omMetadataManager
+ .countRowsInTable(omMetadataManager.getSnapshotInfoTable());
+ assertEquals(expectNumberOfSnapshotAfterPurge,
+ actualNumberOfSnapshotAfterPurge);
+ assertEquals(expectNumberOfSnapshotAfterPurge, chainManager
+ .getGlobalSnapshotChain().size());
validateSnapshotOrderInSnapshotInfoTableAndSnapshotChain(
snapshotInfoListAfterPurge);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
