This is an automated email from the ASF dual-hosted git repository.
sshenoy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 30979b844c HDDS-8909. Support recursive bucket delete using ozone sh
command. (#4981)
30979b844c is described below
commit 30979b844c7f5efac1e2a860668043d28e810f6e
Author: ashishkumar50 <[email protected]>
AuthorDate: Fri Jul 14 11:42:03 2023 +0530
HDDS-8909. Support recursive bucket delete using ozone sh command. (#4981)
---
.../hadoop/ozone/shell/TestOzoneShellHA.java | 113 ++++++++++++++++++
.../apache/hadoop/ozone/shell/OzoneAddress.java | 4 +
.../ozone/shell/bucket/DeleteBucketHandler.java | 128 ++++++++++++++++++++-
3 files changed, 243 insertions(+), 2 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index fa7e9b0275..03d04feba0 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -1293,6 +1293,119 @@ public class TestOzoneShellHA {
Assert.assertEquals(99, getNumOfKeys());
}
+ @Test
+ public void testRecursiveBucketDelete()
+ throws Exception {
+ String volume1 = "volume50";
+ String bucket1 = "bucketfso";
+ String bucket2 = "bucketobs";
+ String bucket3 = "bucketlegacy";
+
+ // Create volume volume1
+ // Create bucket bucket1 with layout FILE_SYSTEM_OPTIMIZED
+ // Insert some keys into it
+ generateKeys(OZONE_URI_DELIMITER + volume1,
+ OZONE_URI_DELIMITER + bucket1,
+ BucketLayout.FILE_SYSTEM_OPTIMIZED.toString());
+
+ // Create OBS bucket in volume1
+ String[] args = new String[] {"bucket", "create", "--layout",
+ BucketLayout.OBJECT_STORE.toString(), volume1 +
+ OZONE_URI_DELIMITER + bucket2};
+ execute(ozoneShell, args);
+ out.reset();
+
+ // Insert few keys into OBS bucket
+ String keyName = OZONE_URI_DELIMITER + volume1 +
+ OZONE_URI_DELIMITER + bucket2 +
+ OZONE_URI_DELIMITER + "key";
+ for (int i = 0; i < 5; i++) {
+ args = new String[] {
+ "key", "put", "o3://" + omServiceId + keyName + i,
+ testFile.getPath()};
+ execute(ozoneShell, args);
+ }
+ out.reset();
+
+ // Create Legacy bucket in volume1
+ args = new String[] {"bucket", "create", "--layout",
+ BucketLayout.LEGACY.toString(), volume1 +
+ OZONE_URI_DELIMITER + bucket3};
+ execute(ozoneShell, args);
+ out.reset();
+
+ // Insert few keys into legacy bucket
+ keyName = OZONE_URI_DELIMITER + volume1 + OZONE_URI_DELIMITER + bucket3 +
+ OZONE_URI_DELIMITER + "key";
+ for (int i = 0; i < 5; i++) {
+ args = new String[] {
+ "key", "put", "o3://" + omServiceId + keyName + i,
+ testFile.getPath()};
+ execute(ozoneShell, args);
+ }
+ out.reset();
+
+ // Try bucket delete without recursive
+ // It should fail as bucket is not empty
+ final String[] args1 = new String[] {"bucket", "delete",
+ volume1 + OZONE_URI_DELIMITER + bucket1};
+ LambdaTestUtils.intercept(ExecutionException.class,
+ "BUCKET_NOT_EMPTY", () -> execute(ozoneShell, args1));
+ out.reset();
+
+ // bucket1 should still exist
+ Assert.assertEquals(client.getObjectStore()
+ .getVolume(volume1).getBucket(bucket1)
+ .getName(), bucket1);
+
+ // Delete bucket1 recursively
+ args =
+ new String[] {"bucket", "delete", volume1 +
+ OZONE_URI_DELIMITER + bucket1, "-r", "--yes"};
+ execute(ozoneShell, args);
+ out.reset();
+
+ // Bucket1 should not exist
+ LambdaTestUtils.intercept(OMException.class,
+ "BUCKET_NOT_FOUND", () -> client.getObjectStore()
+ .getVolume(volume1).getBucket(bucket1));
+
+ // Bucket2 and Bucket3 should still exist
+ Assert.assertEquals(client.getObjectStore().getVolume(volume1)
+ .getBucket(bucket2).getName(), bucket2);
+ Assert.assertEquals(client.getObjectStore().getVolume(volume1)
+ .getBucket(bucket3).getName(), bucket3);
+
+ // Delete bucket2(obs) recursively.
+ args =
+ new String[] {"bucket", "delete", volume1 +
+ OZONE_URI_DELIMITER + bucket2, "-r", "--yes"};
+ execute(ozoneShell, args);
+ out.reset();
+
+ LambdaTestUtils.intercept(OMException.class,
+ "BUCKET_NOT_FOUND", () -> client.getObjectStore()
+ .getVolume(volume1).getBucket(bucket2));
+
+ // Delete bucket3(legacy) recursively.
+ args =
+ new String[] {"bucket", "delete", volume1 +
+ OZONE_URI_DELIMITER + bucket3, "-r", "--yes"};
+ execute(ozoneShell, args);
+ out.reset();
+
+ LambdaTestUtils.intercept(OMException.class,
+ "BUCKET_NOT_FOUND", () -> client.getObjectStore()
+ .getVolume(volume1).getBucket(bucket3));
+
+ // Now delete volume without recursive
+ // All buckets are already deleted
+ args = new String[] {"volume", "delete", volume1};
+ execute(ozoneShell, args);
+ out.reset();
+ LambdaTestUtils.intercept(OMException.class,
+ "VOLUME_NOT_FOUND", () -> client.getObjectStore().getVolume(volume1));
+ }
private void getVolume(String volumeName) {
String[] args = new String[] {"volume", "create",
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java
index cef95dcf0f..e9afce86a4 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/OzoneAddress.java
@@ -297,6 +297,10 @@ public class OzoneAddress {
return bucketName;
}
+ public String getOmHost() {
+ return ozoneURI.getHost();
+ }
+
public String getSnapshotNameWithIndicator() {
return snapshotNameWithIndicator;
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/DeleteBucketHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/DeleteBucketHandler.java
index a8eeee93e1..5626df2082 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/DeleteBucketHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/DeleteBucketHandler.java
@@ -18,30 +18,154 @@
package org.apache.hadoop.ozone.shell.bucket;
+import com.google.common.base.Strings;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Scanner;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
+import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
/**
* Delete bucket Handler.
*/
@Command(name = "delete",
- description = "deletes an empty bucket")
+ description = "deletes a bucket")
public class DeleteBucketHandler extends BucketHandler {
+ @CommandLine.Option(
+ names = {"-r"},
+ description = "Delete bucket recursively"
+ )
+ private boolean bRecursive;
+ @CommandLine.Option(names = {"-y", "--yes"},
+ description = "Continue without interactive user confirmation")
+ private boolean yes;
+ private String omServiceId;
+ private static final int MAX_KEY_DELETE_BATCH_SIZE = 1000;
+
@Override
protected void execute(OzoneClient client, OzoneAddress address)
throws IOException {
String volumeName = address.getVolumeName();
String bucketName = address.getBucketName();
-
OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+
+ if (bRecursive) {
+ if (!Strings.isNullOrEmpty(address.getOmHost())) {
+ omServiceId = address.getOmHost();
+ } else {
+ Collection<String> serviceIds = getConf().getTrimmedStringCollection(
+ OZONE_OM_SERVICE_IDS_KEY);
+ if (serviceIds.size() == 1) {
+ // Only one OM service ID configured, we can use that
+ // If more than 1, it will fail in createClient step itself
+ omServiceId = serviceIds.iterator().next();
+ } else {
+ omServiceId = getConf().get(OZONE_OM_ADDRESS_KEY);
+ }
+ }
+ if (!yes) {
+ // Ask for user confirmation
+ out().print("This command will delete bucket recursively." +
+ "\nThere is no recovery option after using this command, " +
+ "and deleted keys won't move to trash." +
+ "\nEnter 'yes' to proceed': ");
+ out().flush();
+ Scanner scanner = new Scanner(new InputStreamReader(
+ System.in, StandardCharsets.UTF_8));
+ String confirmation = scanner.next().trim().toLowerCase();
+ if (!confirmation.equals("yes")) {
+ out().println("Operation cancelled.");
+ return;
+ }
+ }
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ if (bucket.getBucketLayout().equals(OBJECT_STORE)) {
+ deleteOBSBucketRecursive(vol, bucket);
+ } else {
+ deleteFSBucketRecursive(vol, bucket);
+ }
+ return;
+ }
+ // Delete bucket without recursive
vol.deleteBucket(bucketName);
+ out().printf("Bucket %s is deleted%n", bucketName);
}
+ /**
+ * Delete OBS bucket recursively.
+ *
+ * @param bucket OzoneBucket
+ */
+ private void deleteOBSBucketRecursive(OzoneVolume vol, OzoneBucket bucket) {
+ ArrayList<String> keys = new ArrayList<>();
+ try {
+ if (!bucket.isLink()) {
+ Iterator<? extends OzoneKey> iterator = bucket.listKeys(null);
+ while (iterator.hasNext()) {
+ keys.add(iterator.next().getName());
+ if (MAX_KEY_DELETE_BATCH_SIZE == keys.size()) {
+ bucket.deleteKeys(keys);
+ keys.clear();
+ }
+ }
+ // delete if any remaining keys left
+ if (keys.size() > 0) {
+ bucket.deleteKeys(keys);
+ }
+ }
+ vol.deleteBucket(bucket.getName());
+ out().printf("Bucket %s is deleted%n", bucket.getName());
+ } catch (Exception e) {
+ out().printf("Could not delete bucket %s.%n", e.getMessage());
+ }
+ }
+
+ /**
+ * Delete Legacy/FSO bucket recursively.
+ *
+ * @param vol String
+ * @param bucket OzoneBucket
+ */
+ private void deleteFSBucketRecursive(OzoneVolume vol, OzoneBucket bucket) {
+ try {
+ String hostPrefix = OZONE_OFS_URI_SCHEME + "://" +
+ omServiceId + PATH_SEPARATOR_STR;
+ String ofsPrefix = hostPrefix + vol.getName() + PATH_SEPARATOR_STR +
+ bucket.getName();
+ final Path path = new Path(ofsPrefix);
+ OzoneConfiguration clientConf = new OzoneConfiguration(getConf());
+ clientConf.set(FS_DEFAULT_NAME_KEY, hostPrefix);
+ FileSystem fs = FileSystem.get(clientConf);
+ if (!fs.delete(path, true)) {
+ out().printf("Could not delete bucket %s.%n", bucket.getName());
+ return;
+ }
+ out().printf("Bucket %s is deleted%n", bucket.getName());
+ } catch (IOException e) {
+ out().printf("Could not delete bucket %s.%n", e.getMessage());
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
