sadanand48 commented on code in PR #4981:
URL: https://github.com/apache/ozone/pull/4981#discussion_r1257747562
##########
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/DeleteBucketHandler.java:
##########
@@ -18,30 +18,164 @@
package org.apache.hadoop.ozone.shell.bucket;
+import com.google.common.base.Strings;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Scanner;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
+import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
/**
* Delete bucket Handler.
*/
@Command(name = "delete",
- description = "deletes an empty bucket")
+ description = "deletes a bucket")
public class DeleteBucketHandler extends BucketHandler {
+ @CommandLine.Option(
+ names = {"-r"},
+ description = "Delete bucket recursively"
+ )
+ private boolean bRecursive;
+
+ @CommandLine.Option(
+ names = {"-id", "--om-service-id"},
+ description = "Ozone Manager Service ID"
+ )
+ private String omServiceId;
+
+ @CommandLine.Option(names = {"-y", "--yes"},
+ description = "Continue without interactive user confirmation")
+ private boolean yes;
+
+ private static final int MAX_KEY_DELETE_BATCH_SIZE = 1000;
+
@Override
protected void execute(OzoneClient client, OzoneAddress address)
throws IOException {
String volumeName = address.getVolumeName();
String bucketName = address.getBucketName();
-
OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+
+ if (bRecursive) {
+ Collection<String> serviceIds = getConf().getTrimmedStringCollection(
+ OZONE_OM_SERVICE_IDS_KEY);
+ if (Strings.isNullOrEmpty(omServiceId)) {
+ if (serviceIds.size() > 1) {
+ out().printf("OmServiceID not provided, provide using " +
+ "-id <OM_SERVICE_ID>%n");
+ return;
+ } else if (serviceIds.size() == 1) {
+ // Only one OM service ID configured, we can use that
+ omServiceId = serviceIds.iterator().next();
+ }
+ }
+ if (!yes) {
+ // Ask for user confirmation
+ out().print("This command will delete bucket recursively." +
+ "\nThere is no recovery option after using this command, " +
+ "and no trash for FSO buckets." +
Review Comment:
I guess " and deleted keys won't move to trash " might be better.
##########
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/DeleteBucketHandler.java:
##########
@@ -18,30 +18,164 @@
package org.apache.hadoop.ozone.shell.bucket;
+import com.google.common.base.Strings;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Scanner;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
+import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
/**
* Delete bucket Handler.
*/
@Command(name = "delete",
- description = "deletes an empty bucket")
+ description = "deletes a bucket")
public class DeleteBucketHandler extends BucketHandler {
+ @CommandLine.Option(
+ names = {"-r"},
+ description = "Delete bucket recursively"
+ )
+ private boolean bRecursive;
+
+ @CommandLine.Option(
+ names = {"-id", "--om-service-id"},
+ description = "Ozone Manager Service ID"
+ )
+ private String omServiceId;
+
+ @CommandLine.Option(names = {"-y", "--yes"},
+ description = "Continue without interactive user confirmation")
+ private boolean yes;
+
+ private static final int MAX_KEY_DELETE_BATCH_SIZE = 1000;
+
@Override
protected void execute(OzoneClient client, OzoneAddress address)
throws IOException {
String volumeName = address.getVolumeName();
String bucketName = address.getBucketName();
-
OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+
+ if (bRecursive) {
+ Collection<String> serviceIds = getConf().getTrimmedStringCollection(
+ OZONE_OM_SERVICE_IDS_KEY);
+ if (Strings.isNullOrEmpty(omServiceId)) {
+ if (serviceIds.size() > 1) {
+ out().printf("OmServiceID not provided, provide using " +
+ "-id <OM_SERVICE_ID>%n");
+ return;
+ } else if (serviceIds.size() == 1) {
+ // Only one OM service ID configured, we can use that
+ omServiceId = serviceIds.iterator().next();
+ }
+ }
+ if (!yes) {
+ // Ask for user confirmation
+ out().print("This command will delete bucket recursively." +
+ "\nThere is no recovery option after using this command, " +
+ "and no trash for FSO buckets." +
+ "\nEnter 'yes' to proceed': ");
+ out().flush();
+ Scanner scanner = new Scanner(new InputStreamReader(
+ System.in, StandardCharsets.UTF_8));
+ String confirmation = scanner.next().trim().toLowerCase();
+ if (!confirmation.equals("yes")) {
+ out().println("Operation cancelled.");
+ return;
+ }
+ }
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ if (bucket.getBucketLayout().equals(OBJECT_STORE)) {
+ deleteOBSBucketRecursive(vol, bucket);
+ } else {
+ deleteFSBucketRecursive(vol, bucket);
+ }
+ return;
+ }
+ // Delete bucket without recursive
vol.deleteBucket(bucketName);
+ out().printf("Bucket %s is deleted%n", bucketName);
}
+ /**
+ * Delete OBS bucket recursively.
+ *
+ * @param bucket OzoneBucket
+ */
+ private void deleteOBSBucketRecursive(OzoneVolume vol, OzoneBucket bucket) {
+ ArrayList<String> keys = new ArrayList<>();
+ try {
+ if (!bucket.isLink()) {
+ Iterator<? extends OzoneKey> iterator = bucket.listKeys(null);
+ while (iterator.hasNext()) {
+ keys.add(iterator.next().getName());
+ if (MAX_KEY_DELETE_BATCH_SIZE == keys.size()) {
+ bucket.deleteKeys(keys);
+ keys.clear();
+ }
+ }
+ // delete if any remaining keys left
+ if (keys.size() > 0) {
+ bucket.deleteKeys(keys);
+ }
+ }
+ vol.deleteBucket(bucket.getName());
+ out().printf("Bucket %s is deleted%n", bucket.getName());
+ } catch (Exception e) {
+ out().println("Could not delete bucket.");
+ }
+ }
+
+ /**
+ * Delete Legacy/FSO bucket recursively.
+ *
+ * @param vol String
+ * @param bucket OzoneBucket
+ */
+ private void deleteFSBucketRecursive(OzoneVolume vol, OzoneBucket bucket) {
+ try {
+ String hostPrefix = OZONE_OFS_URI_SCHEME + "://";
+ if (!Strings.isNullOrEmpty(omServiceId)) {
Review Comment:
omServiceId is always nonNull/nonEmpty because of the earlier check on line
85. Checking it here again is not required.
##########
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/DeleteBucketHandler.java:
##########
@@ -18,30 +18,164 @@
package org.apache.hadoop.ozone.shell.bucket;
+import com.google.common.base.Strings;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.shell.OzoneAddress;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Scanner;
+
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.PATH_SEPARATOR_STR;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
+import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
/**
* Delete bucket Handler.
*/
@Command(name = "delete",
- description = "deletes an empty bucket")
+ description = "deletes a bucket")
public class DeleteBucketHandler extends BucketHandler {
+ @CommandLine.Option(
+ names = {"-r"},
+ description = "Delete bucket recursively"
+ )
+ private boolean bRecursive;
+
+ @CommandLine.Option(
+ names = {"-id", "--om-service-id"},
+ description = "Ozone Manager Service ID"
+ )
+ private String omServiceId;
+
+ @CommandLine.Option(names = {"-y", "--yes"},
+ description = "Continue without interactive user confirmation")
+ private boolean yes;
+
+ private static final int MAX_KEY_DELETE_BATCH_SIZE = 1000;
+
@Override
protected void execute(OzoneClient client, OzoneAddress address)
throws IOException {
String volumeName = address.getVolumeName();
String bucketName = address.getBucketName();
-
OzoneVolume vol = client.getObjectStore().getVolume(volumeName);
+
+ if (bRecursive) {
+ Collection<String> serviceIds = getConf().getTrimmedStringCollection(
+ OZONE_OM_SERVICE_IDS_KEY);
+ if (Strings.isNullOrEmpty(omServiceId)) {
+ if (serviceIds.size() > 1) {
+ out().printf("OmServiceID not provided, provide using " +
+ "-id <OM_SERVICE_ID>%n");
+ return;
+ } else if (serviceIds.size() == 1) {
+ // Only one OM service ID configured, we can use that
+ omServiceId = serviceIds.iterator().next();
+ }
+ }
+ if (!yes) {
+ // Ask for user confirmation
+ out().print("This command will delete bucket recursively." +
+ "\nThere is no recovery option after using this command, " +
+ "and no trash for FSO buckets." +
+ "\nEnter 'yes' to proceed': ");
+ out().flush();
+ Scanner scanner = new Scanner(new InputStreamReader(
+ System.in, StandardCharsets.UTF_8));
+ String confirmation = scanner.next().trim().toLowerCase();
+ if (!confirmation.equals("yes")) {
+ out().println("Operation cancelled.");
+ return;
+ }
+ }
+ OzoneBucket bucket = vol.getBucket(bucketName);
+ if (bucket.getBucketLayout().equals(OBJECT_STORE)) {
+ deleteOBSBucketRecursive(vol, bucket);
+ } else {
+ deleteFSBucketRecursive(vol, bucket);
+ }
+ return;
+ }
+ // Delete bucket without recursive
vol.deleteBucket(bucketName);
+ out().printf("Bucket %s is deleted%n", bucketName);
}
+ /**
+ * Delete OBS bucket recursively.
+ *
+ * @param bucket OzoneBucket
+ */
+ private void deleteOBSBucketRecursive(OzoneVolume vol, OzoneBucket bucket) {
+ ArrayList<String> keys = new ArrayList<>();
+ try {
+ if (!bucket.isLink()) {
Review Comment:
IMO deleting a bucket link should only delete the link & not the source
bucket & its contents.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
