This is an automated email from the ASF dual-hosted git repository.
prashantpogde pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 9b59ade990 HDDS-8685. [SNAPSHOT] CreateCheckpoint needs to be
idempotent. (#4946)
9b59ade990 is described below
commit 9b59ade990a87a5b8034dcc18be1e997b4f59578
Author: Aswin Shakil Balasubramanian <[email protected]>
AuthorDate: Wed Jun 21 23:04:02 2023 -0700
HDDS-8685. [SNAPSHOT] CreateCheckpoint needs to be idempotent. (#4946)
---
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 24 ++++++++--
.../hadoop/ozone/om/TestOmSnapshotManager.java | 51 ++++++++++++++++++++++
2 files changed, 72 insertions(+), 3 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 3dd07b002b..a1dbd2eaa7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -27,6 +27,8 @@ import com.google.common.cache.RemovalListener;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
@@ -44,6 +46,7 @@ import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.hdds.utils.db.CodecRegistry;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.hdds.utils.db.RDBStore;
+import org.apache.hadoop.hdds.utils.db.RocksDBCheckpoint;
import org.apache.hadoop.hdds.utils.db.RocksDatabase;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
@@ -99,7 +102,7 @@ import static
org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.dropColumnFamily
*/
public final class OmSnapshotManager implements AutoCloseable {
public static final String OM_HARDLINK_FILE = "hardLinkFile";
- private static final Logger LOG =
+ public static final Logger LOG =
LoggerFactory.getLogger(OmSnapshotManager.class);
// Threshold for the table iterator loop in nanoseconds.
@@ -395,9 +398,20 @@ public final class OmSnapshotManager implements
AutoCloseable {
omMetadataManager.getTableLock(OmMetadataManagerImpl.DELETED_TABLE)
.writeLock().lock();
+ boolean snapshotDirExist = false;
+
try {
// Create DB checkpoint for snapshot
- dbCheckpoint = store.getSnapshot(snapshotInfo.getCheckpointDirName());
+ String checkpointPrefix = store.getDbLocation().getName();
+ Path snapshotDirPath = Paths.get(store.getSnapshotsParentDir(),
+ checkpointPrefix + snapshotInfo.getCheckpointDir());
+ if (Files.exists(snapshotDirPath)) {
+ snapshotDirExist = true;
+ dbCheckpoint = new RocksDBCheckpoint(snapshotDirPath);
+ } else {
+ dbCheckpoint = store.getSnapshot(snapshotInfo.getCheckpointDirName());
+ }
+
// Clean up active DB's deletedTable right after checkpoint is taken,
// with table write lock held
deleteKeysFromDelKeyTableInSnapshotScope(omMetadataManager,
@@ -414,7 +428,11 @@ public final class OmSnapshotManager implements
AutoCloseable {
.writeLock().unlock();
}
- if (dbCheckpoint != null) {
+ if (dbCheckpoint != null && snapshotDirExist) {
+ LOG.info("Checkpoint : {} for snapshot {} already exists.",
+ dbCheckpoint.getCheckpointLocation(), snapshotInfo.getName());
+ return dbCheckpoint;
+ } else if (dbCheckpoint != null) {
LOG.info("Created checkpoint : {} for snapshot {}",
dbCheckpoint.getCheckpointLocation(), snapshotInfo.getName());
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
index 445bdb2a4b..d5887126de 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java
@@ -386,6 +386,57 @@ public class TestOmSnapshotManager {
Assert.assertTrue(copyFiles.contains(addNonSstToCopiedFiles));
}
+ @Test
+ public void testCreateSnapshotIdempotent() throws Exception {
+ // set up db tables
+ GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer
+ .captureLogs(OmSnapshotManager.LOG);
+ Table<String, OmVolumeArgs> volumeTable = mock(Table.class);
+ Table<String, OmBucketInfo> bucketTable = mock(Table.class);
+ Table<String, SnapshotInfo> snapshotInfoTable = mock(Table.class);
+ HddsWhiteboxTestUtils.setInternalState(
+ om.getMetadataManager(), VOLUME_TABLE, volumeTable);
+ HddsWhiteboxTestUtils.setInternalState(
+ om.getMetadataManager(), BUCKET_TABLE, bucketTable);
+ HddsWhiteboxTestUtils.setInternalState(
+ om.getMetadataManager(), SNAPSHOT_INFO_TABLE, snapshotInfoTable);
+
+ final String volumeName = UUID.randomUUID().toString();
+ final String dbVolumeKey =
om.getMetadataManager().getVolumeKey(volumeName);
+ final OmVolumeArgs omVolumeArgs = OmVolumeArgs.newBuilder()
+ .setVolume(volumeName)
+ .setAdminName("bilbo")
+ .setOwnerName("bilbo")
+ .build();
+ when(volumeTable.get(dbVolumeKey)).thenReturn(omVolumeArgs);
+
+ String bucketName = UUID.randomUUID().toString();
+ final String dbBucketKey = om.getMetadataManager().getBucketKey(
+ volumeName, bucketName);
+ final OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .build();
+ when(bucketTable.get(dbBucketKey)).thenReturn(omBucketInfo);
+
+ SnapshotInfo first = createSnapshotInfo(volumeName, bucketName);
+ when(snapshotInfoTable.get(first.getTableKey())).thenReturn(first);
+
+ // Create first checkpoint for the snapshot checkpoint
+ OmSnapshotManager.createOmSnapshotCheckpoint(om.getMetadataManager(),
+ first);
+ Assert.assertFalse(logCapturer.getOutput().contains(
+ "for snapshot " + first.getName() + " already exists."));
+ logCapturer.clearOutput();
+
+ // Create checkpoint again for the same snapshot.
+ OmSnapshotManager.createOmSnapshotCheckpoint(om.getMetadataManager(),
+ first);
+
+ Assert.assertTrue(logCapturer.getOutput().contains(
+ "for snapshot " + first.getName() + " already exists."));
+ }
+
private SnapshotInfo createSnapshotInfo(String volumeName,
String bucketName) {
return SnapshotInfo.newInstance(volumeName,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
