This is an automated email from the ASF dual-hosted git repository.
sshenoy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a248ed1ea4 HDDS-10329. [Snapshot] Add unit-test for recreating
snapshots with deleted snapshot names. (#6298)
a248ed1ea4 is described below
commit a248ed1ea421770bb22fa3ce58906194f793406c
Author: jyotirmoy-gh <[email protected]>
AuthorDate: Tue Mar 5 10:23:06 2024 +0530
HDDS-10329. [Snapshot] Add unit-test for recreating snapshots with deleted
snapshot names. (#6298)
---
.../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 83 +++++++++++++++++++---
.../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 59 ++++++++++++++-
2 files changed, 132 insertions(+), 10 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java
index 7afdf7144f..8e0bd1ac7d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java
@@ -24,12 +24,14 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.om.KeyManagerImpl;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
@@ -47,9 +49,11 @@ import org.junit.jupiter.params.provider.ValueSource;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -66,12 +70,13 @@ class TestOzoneFsSnapshot {
private static final String OM_SERVICE_ID = "om-service-test1";
private static OzoneManager ozoneManager;
private static OzoneFsShell shell;
+ private static AtomicInteger counter = new AtomicInteger();
private static final String VOLUME =
- "vol-" + RandomStringUtils.randomNumeric(5);
+ "vol-" + counter.incrementAndGet();
private static final String BUCKET =
- "buck-" + RandomStringUtils.randomNumeric(5);
+ "buck-" + counter.incrementAndGet();
private static final String KEY =
- "key-" + RandomStringUtils.randomNumeric(5);
+ "key-" + counter.incrementAndGet();
private static final String BUCKET_PATH =
OM_KEY_PREFIX + VOLUME + OM_KEY_PREFIX + BUCKET;
private static final String BUCKET_WITH_SNAPSHOT_INDICATOR_PATH =
@@ -84,6 +89,8 @@ class TestOzoneFsSnapshot {
OzoneConfiguration conf = new OzoneConfiguration();
// Enable filesystem snapshot feature for the test regardless of the
default
conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true);
+ conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1,
TimeUnit.SECONDS);
+ conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL,
KeyManagerImpl.DISABLE_VALUE);
// Start the cluster
cluster = MiniOzoneCluster.newHABuilder(conf)
@@ -128,7 +135,7 @@ class TestOzoneFsSnapshot {
@Test
void testCreateSnapshotDuplicateName() throws Exception {
- String snapshotName = "snap-" + RandomStringUtils.randomNumeric(5);
+ String snapshotName = "snap-" + counter.incrementAndGet();
int res = ToolRunner.run(shell,
new String[]{"-createSnapshot", BUCKET_PATH, snapshotName});
@@ -152,7 +159,7 @@ class TestOzoneFsSnapshot {
// rather than:
// Created snapshot ofs://om/vol1/buck2/dir3/.snapshot/snap1
- String snapshotName = "snap-" + RandomStringUtils.randomNumeric(5);
+ String snapshotName = "snap-" + counter.incrementAndGet();
String dirPath = BUCKET_PATH + "/dir1/";
@@ -257,7 +264,7 @@ class TestOzoneFsSnapshot {
*/
@Test
void testFsLsSnapshot(@TempDir Path tempDir) throws Exception {
- String key1 = "key-" + RandomStringUtils.randomNumeric(5);
+ String key1 = "key-" + counter.incrementAndGet();
String newKeyPath = BUCKET_PATH + OM_KEY_PREFIX + key1;
// Pause SnapshotDeletingService so that Snapshot marked deleted is not
reclaimed.
ozoneManager.getKeyManager().getSnapshotDeletingService().suspend();
@@ -274,7 +281,7 @@ class TestOzoneFsSnapshot {
String snapshotPath1 = BUCKET_WITH_SNAPSHOT_INDICATOR_PATH +
OM_KEY_PREFIX + snapshotName1;
- String key2 = "key-" + RandomStringUtils.randomNumeric(5);
+ String key2 = "key-" + counter.incrementAndGet();
String newKeyPath2 = BUCKET_PATH + OM_KEY_PREFIX + key2;
execShellCommandAndGetOutput(0,
new String[]{"-put", tempFile.toString(), newKeyPath2});
@@ -413,6 +420,64 @@ class TestOzoneFsSnapshot {
assertThat(errorMessage).contains(expectedMessage);
}
+ @Test
+ public void testSnapshotReuseSnapName() throws Exception {
+ String key1 = "key-" + counter.incrementAndGet();
+ int res = ToolRunner.run(shell, new String[]{"-touch",
+ BUCKET_PATH + OM_KEY_PREFIX + key1});
+ assertEquals(0, res);
+
+ String snap1 = "snap" + counter.incrementAndGet();
+ res = ToolRunner.run(shell,
+ new String[]{"-createSnapshot", BUCKET_PATH, snap1});
+ // Asserts that create request succeeded
+ assertEquals(0, res);
+
+ String listSnapOut = execShellCommandAndGetOutput(0,
+ new String[]{"-ls", BUCKET_WITH_SNAPSHOT_INDICATOR_PATH +
OM_KEY_PREFIX + snap1});
+ assertThat(listSnapOut).contains(key1);
+
+ res = ToolRunner.run(shell,
+ new String[]{"-deleteSnapshot", BUCKET_PATH, snap1});
+ // Asserts that delete request succeeded
+ assertEquals(0, res);
+
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return !ozoneManager.getMetadataManager().getSnapshotInfoTable()
+ .isExist(SnapshotInfo.getTableKey(VOLUME, BUCKET, snap1));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }, 200, 10000);
+
+ String key2 = "key-" + counter.incrementAndGet();
+ res = ToolRunner.run(shell, new String[]{"-touch",
+ BUCKET_PATH + OM_KEY_PREFIX + key2});
+ assertEquals(0, res);
+ String snap2 = "snap" + counter.incrementAndGet();
+ res = ToolRunner.run(shell,
+ new String[]{"-createSnapshot", BUCKET_PATH, snap2});
+ // Asserts that create request succeeded
+ assertEquals(0, res);
+
+ String key3 = "key-" + counter.incrementAndGet();
+ res = ToolRunner.run(shell, new String[]{"-touch",
+ BUCKET_PATH + OM_KEY_PREFIX + key3});
+ assertEquals(0, res);
+
+ res = ToolRunner.run(shell,
+ new String[]{"-createSnapshot", BUCKET_PATH, snap1});
+ // Asserts that create request succeeded
+ assertEquals(0, res);
+
+ listSnapOut = execShellCommandAndGetOutput(0,
+ new String[]{"-ls", BUCKET_WITH_SNAPSHOT_INDICATOR_PATH +
OM_KEY_PREFIX + snap1});
+ assertThat(listSnapOut).contains(key1);
+ assertThat(listSnapOut).contains(key2);
+ assertThat(listSnapOut).contains(key3);
+ }
+
/**
* Execute a shell command with provided arguments
* and return a string of the output.
@@ -453,7 +518,7 @@ class TestOzoneFsSnapshot {
}
private String createSnapshot() throws Exception {
- String snapshotName = "snap-" + RandomStringUtils.randomNumeric(5);
+ String snapshotName = "snap-" + counter.incrementAndGet();
// Create snapshot
int res = ToolRunner.run(shell,
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
index 8021b95984..b4e06c0342 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshot.java
@@ -102,6 +102,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Matcher;
@@ -111,6 +112,7 @@ import java.util.stream.Collectors;
import static org.apache.commons.lang3.StringUtils.leftPad;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DB_PROFILE;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.DEFAULT;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL;
import static
org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isDone;
import static
org.apache.hadoop.ozone.admin.scm.FinalizeUpgradeCommandUtil.isStarting;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
@@ -118,6 +120,7 @@ import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOU
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DISABLE_NATIVE_LIBS;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER;
import static
org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
@@ -172,6 +175,7 @@ public abstract class TestOmSnapshot {
private ObjectStore store;
private OzoneManager ozoneManager;
private OzoneBucket ozoneBucket;
+ private OzoneConfiguration conf;
private final BucketLayout bucketLayout;
private final boolean enabledFileSystemPaths;
@@ -196,7 +200,7 @@ public abstract class TestOmSnapshot {
* Create a MiniDFSCluster for testing.
*/
private void init() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
+ conf = new OzoneConfiguration();
conf.setBoolean(OZONE_OM_ENABLE_FILESYSTEM_PATHS, enabledFileSystemPaths);
conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, bucketLayout.name());
conf.setBoolean(OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF, forceFullSnapshotDiff);
@@ -209,6 +213,8 @@ public abstract class TestOmSnapshot {
// Enable filesystem snapshot feature for the test regardless of the
default
conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true);
conf.setInt(OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION,
OMLayoutFeature.BUCKET_LAYOUT_SUPPORT.layoutVersion());
+ conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1,
TimeUnit.SECONDS);
+ conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL,
KeyManagerImpl.DISABLE_VALUE);
cluster = MiniOzoneCluster.newBuilder(conf)
.build();
@@ -236,6 +242,12 @@ public abstract class TestOmSnapshot {
keyManager.stop();
}
+ private void startKeyManager() throws IOException {
+ KeyManagerImpl keyManager = (KeyManagerImpl) HddsWhiteboxTestUtils
+ .getInternalState(ozoneManager, "keyManager");
+ keyManager.start(conf);
+ }
+
private RDBStore getRdbStore() {
return (RDBStore) ozoneManager.getMetadataManager().getStore();
}
@@ -2481,4 +2493,49 @@ public abstract class TestOmSnapshot {
fetchReportPage(volume1, bucket3, "bucket3-snap1", "bucket3-snap3",
null, 0).getDiffList().size());
}
+
+ @Test
+ public void testSnapshotReuseSnapName() throws Exception {
+ // start KeyManager for this test
+ startKeyManager();
+ String volume = "vol-" + counter.incrementAndGet();
+ String bucket = "buck-" + counter.incrementAndGet();
+ store.createVolume(volume);
+ OzoneVolume volume1 = store.getVolume(volume);
+ volume1.createBucket(bucket);
+ OzoneBucket bucket1 = volume1.getBucket(bucket);
+ // Create Key1 and take snapshot
+ String key1 = "key-1-";
+ createFileKeyWithPrefix(bucket1, key1);
+ String snap1 = "snap" + counter.incrementAndGet();
+ String snapshotKeyPrefix = createSnapshot(volume, bucket, snap1);
+
+ int keyCount1 = keyCount(bucket1, snapshotKeyPrefix + "key-");
+ assertEquals(1, keyCount1);
+
+ store.deleteSnapshot(volume, bucket, snap1);
+
+ GenericTestUtils.waitFor(() -> {
+ try {
+ return !ozoneManager.getMetadataManager().getSnapshotInfoTable()
+ .isExist(SnapshotInfo.getTableKey(volume, bucket, snap1));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }, 200, 10000);
+
+ createFileKeyWithPrefix(bucket1, key1);
+ String snap2 = "snap" + counter.incrementAndGet();
+ createSnapshot(volume, bucket, snap2);
+
+ String key2 = "key-2-";
+ createFileKeyWithPrefix(bucket1, key2);
+ createSnapshot(volume, bucket, snap1);
+
+ int keyCount2 = keyCount(bucket1, snapshotKeyPrefix + "key-");
+ assertEquals(3, keyCount2);
+
+ // Stop key manager after testcase executed
+ stopKeyManager();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
