This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 44adf80324 HDDS-10359. Recursively deleting volume with OBS bucket
shows error despite success (#6217)
44adf80324 is described below
commit 44adf80324f85853b98ddbf2e2ce0a074f354f3c
Author: ashishkumar50 <[email protected]>
AuthorDate: Wed Feb 14 14:10:40 2024 +0530
HDDS-10359. Recursively deleting volume with OBS bucket shows error despite
success (#6217)
---
.../hadoop/ozone/shell/TestOzoneShellHA.java | 92 ++++++++++++----------
.../ozone/shell/volume/DeleteVolumeHandler.java | 6 ++
2 files changed, 55 insertions(+), 43 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index 9596e96026..51956accb1 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -82,6 +82,9 @@ import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCK
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_EMPTY;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
+import static
org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
+import static org.apache.hadoop.ozone.om.helpers.BucketLayout.LEGACY;
+import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -97,6 +100,8 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import picocli.CommandLine;
@@ -1836,9 +1841,10 @@ public class TestOzoneShellHA {
OMException exception = (OMException) execution.getCause();
assertEquals(VOLUME_NOT_FOUND, exception.getResult());
}
-
- @Test
- public void testRecursiveVolumeDelete()
+
+ @ParameterizedTest
+ @ValueSource(ints = {1, 5})
+ public void testRecursiveVolumeDelete(int threadCount)
throws Exception {
String volume1 = "volume10";
String volume2 = "volume20";
@@ -1847,47 +1853,19 @@ public class TestOzoneShellHA {
// Create bucket bucket1 with layout FILE_SYSTEM_OPTIMIZED
// Insert some keys into it
generateKeys(OZONE_URI_DELIMITER + volume1,
- "/bucketfso",
+ "/fsobucket1",
BucketLayout.FILE_SYSTEM_OPTIMIZED.toString());
- // Create another volume volume2 with bucket and some keys into it.
+ // Create another volume volume2 with bucket and some keys into it.
generateKeys(OZONE_URI_DELIMITER + volume2,
"/bucket2",
BucketLayout.FILE_SYSTEM_OPTIMIZED.toString());
- // Create OBS bucket in volume1
- String[] args = new String[] {"bucket", "create", "--layout",
- BucketLayout.OBJECT_STORE.toString(), volume1 + "/bucketobs"};
- execute(ozoneShell, args);
- out.reset();
-
- // Insert few keys into OBS bucket
- String keyName = OZONE_URI_DELIMITER + volume1 + "/bucketobs" +
- OZONE_URI_DELIMITER + "key";
- for (int i = 0; i < 5; i++) {
- args = new String[] {
- "key", "put", "o3://" + omServiceId + keyName + i,
- testFile.getPath()};
- execute(ozoneShell, args);
- }
- out.reset();
-
- // Create Legacy bucket in volume1
- args = new String[] {"bucket", "create", "--layout",
- BucketLayout.LEGACY.toString(), volume1 + "/bucketlegacy"};
- execute(ozoneShell, args);
- out.reset();
-
- // Insert few keys into legacy bucket
- keyName = OZONE_URI_DELIMITER + volume1 + "/bucketlegacy" +
- OZONE_URI_DELIMITER + "key";
- for (int i = 0; i < 5; i++) {
- args = new String[] {
- "key", "put", "o3://" + omServiceId + keyName + i,
- testFile.getPath()};
- execute(ozoneShell, args);
- }
- out.reset();
+ createBucketAndGenerateKeys(volume1, FILE_SYSTEM_OPTIMIZED, "fsobucket2");
+ createBucketAndGenerateKeys(volume1, OBJECT_STORE, "obsbucket1");
+ createBucketAndGenerateKeys(volume1, OBJECT_STORE, "obsbucket2");
+ createBucketAndGenerateKeys(volume1, LEGACY, "legacybucket1");
+ createBucketAndGenerateKeys(volume1, LEGACY, "legacybucket2");
// Try volume delete without recursive
// It should fail as volume is not empty
@@ -1902,22 +1880,50 @@ public class TestOzoneShellHA {
assertEquals(client.getObjectStore().getVolume(volume1)
.getName(), volume1);
- // Delete volume1(containing OBS, FSO and Legacy buckets) recursively
- args =
- new String[] {"volume", "delete", volume1, "-r", "--yes"};
+ // Delete volume1(containing OBS, FSO and Legacy buckets) recursively with
thread count
+ String[] args = new String[] {"volume", "delete", volume1, "-r", "--yes",
"-t", String.valueOf(threadCount)};
execute(ozoneShell, args);
out.reset();
+ // volume1 should not exist
+ omExecution = assertThrows(OMException.class,
+ () -> client.getObjectStore().getVolume(volume1));
+ assertEquals(VOLUME_NOT_FOUND, omExecution.getResult());
+
// volume2 should still exist
assertEquals(client.getObjectStore().getVolume(volume2)
.getName(), volume2);
- // volume1 should not exist
+ // Delete volume2 recursively
+ args = new String[] {"volume", "delete", volume2, "-r", "--yes"};
+ execute(ozoneShell, args);
+ out.reset();
+
+ // volume2 should not exist
omExecution = assertThrows(OMException.class,
- () -> client.getObjectStore().getVolume(volume1));
+ () -> client.getObjectStore().getVolume(volume2));
assertEquals(VOLUME_NOT_FOUND, omExecution.getResult());
}
+ private void createBucketAndGenerateKeys(String volume, BucketLayout layout,
String bucketName) {
+ // Create bucket
+ String[] args = new String[] {"bucket", "create", volume + "/" +
bucketName,
+ "--layout", layout.toString()};
+ execute(ozoneShell, args);
+ out.reset();
+
+ // Insert keys
+ String keyName = OZONE_URI_DELIMITER + volume + "/" + bucketName +
+ OZONE_URI_DELIMITER + "key";
+ for (int i = 0; i < 5; i++) {
+ args = new String[] {
+ "key", "put", "o3://" + omServiceId + keyName + i,
+ testFile.getPath()};
+ execute(ozoneShell, args);
+ }
+ out.reset();
+ }
+
@Test
public void testLinkedAndNonLinkedBucketMetaData()
throws Exception {
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java
index e380e98561..8cc8050238 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/DeleteVolumeHandler.java
@@ -121,6 +121,11 @@ public class DeleteVolumeHandler extends VolumeHandler {
totalBucketCount++;
}
doCleanBuckets();
+ // Reset counters and bucket list
+ numberOfBucketsCleaned.set(0);
+ totalBucketCount = 0;
+ cleanedBucketCounter.set(0);
+ bucketIdList.clear();
}
/**
@@ -201,6 +206,7 @@ public class DeleteVolumeHandler extends VolumeHandler {
if (!cleanOBSBucket(bucket)) {
throw new RuntimeException("Failed to clean bucket");
}
+ break;
default:
throw new RuntimeException("Invalid bucket layout");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
