This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/HDDS-2665-ofs by this push:
new 0773a4a HDDS-3494. Implement ofs://: Support volume and bucket
deletion (#906)
0773a4a is described below
commit 0773a4aef90d27720c89cdc34dfcba5b0f7de5d5
Author: Siyao Meng <[email protected]>
AuthorDate: Mon May 18 09:55:45 2020 -0700
HDDS-3494. Implement ofs://: Support volume and bucket deletion (#906)
---
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 107 +++++++++++++++++++++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 7 ++
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 7 ++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 75 +++++++++++++--
.../java/org/apache/hadoop/fs/ozone/OFSPath.java | 11 +++
5 files changed, 199 insertions(+), 8 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index b0550e1..4312905 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -65,6 +66,7 @@ import static
org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
+import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
/**
* Ozone file system tests that are not covered by contract tests.
@@ -766,4 +768,109 @@ public class TestRootedOzoneFileSystem {
fileStatusesInDir1[0].getPath().toUri().getPath());
}
+ /**
+ * Helper function. Check Ozone volume existence.
+ * @param volumeStr Name of the volume
+ * @return true if volume exists, false if not
+ */
+ private boolean volumeExist(String volumeStr) throws IOException {
+ try {
+ objectStore.getVolume(volumeStr);
+ } catch (OMException ex) {
+ if (ex.getResult() == VOLUME_NOT_FOUND) {
+ return false;
+ } else {
+ throw ex;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Helper function. Delete a path non-recursively and expect failure.
+ * @param f Path to delete.
+ * @throws IOException
+ */
+ private void deleteNonRecursivelyAndFail(Path f) throws IOException {
+ try {
+ fs.delete(f, false);
+ Assert.fail("Should have thrown PathIsNotEmptyDirectoryException!");
+ } catch (PathIsNotEmptyDirectoryException ignored) {
+ }
+ }
+
+ @Test
+ public void testDeleteEmptyVolume() throws IOException {
+ // Create volume
+ String volumeStr1 = getRandomNonExistVolumeName();
+ Path volumePath1 = new Path(OZONE_URI_DELIMITER + volumeStr1);
+ fs.mkdirs(volumePath1);
+ // Check volume creation
+ OzoneVolume volume1 = objectStore.getVolume(volumeStr1);
+ Assert.assertEquals(volumeStr1, volume1.getName());
+ // Delete empty volume non-recursively
+ Assert.assertTrue(fs.delete(volumePath1, false));
+ // Verify the volume is deleted
+ Assert.assertFalse(volumeStr1 + " should have been deleted!",
+ volumeExist(volumeStr1));
+ }
+
+ @Test
+ public void testDeleteVolumeAndBucket() throws IOException {
+ // Create volume and bucket
+ String volumeStr2 = getRandomNonExistVolumeName();
+ Path volumePath2 = new Path(OZONE_URI_DELIMITER + volumeStr2);
+ String bucketStr2 = "bucket2";
+ Path bucketPath2 = new Path(volumePath2, bucketStr2);
+ fs.mkdirs(bucketPath2);
+ // Check volume and bucket creation
+ OzoneVolume volume2 = objectStore.getVolume(volumeStr2);
+ Assert.assertEquals(volumeStr2, volume2.getName());
+ OzoneBucket bucket2 = volume2.getBucket(bucketStr2);
+ Assert.assertEquals(bucketStr2, bucket2.getName());
+ // Delete volume non-recursively should fail since it is not empty
+ deleteNonRecursivelyAndFail(volumePath2);
+ // Delete bucket first, then volume
+ Assert.assertTrue(fs.delete(bucketPath2, false));
+ Assert.assertTrue(fs.delete(volumePath2, false));
+ // Verify the volume is deleted
+ Assert.assertFalse(volumeExist(volumeStr2));
+ }
+
+ @Test
+ public void testDeleteVolumeBucketAndKey() throws IOException {
+ // Create test volume, bucket and key
+ String volumeStr3 = getRandomNonExistVolumeName();
+ Path volumePath3 = new Path(OZONE_URI_DELIMITER + volumeStr3);
+ String bucketStr3 = "bucket3";
+ Path bucketPath3 = new Path(volumePath3, bucketStr3);
+ String dirStr3 = "dir3";
+ Path dirPath3 = new Path(bucketPath3, dirStr3);
+ fs.mkdirs(dirPath3);
+ // Delete volume or bucket non-recursively, should fail
+ deleteNonRecursivelyAndFail(volumePath3);
+ deleteNonRecursivelyAndFail(bucketPath3);
+ // Delete key first, then bucket, then volume
+ Assert.assertTrue(fs.delete(dirPath3, false));
+ Assert.assertTrue(fs.delete(bucketPath3, false));
+ Assert.assertTrue(fs.delete(volumePath3, false));
+ // Verify the volume is deleted
+ Assert.assertFalse(volumeExist(volumeStr3));
+
+ // Test recursively delete volume
+ // Create test volume, bucket and key
+ fs.mkdirs(dirPath3);
+ // Delete volume recursively
+ Assert.assertTrue(fs.delete(volumePath3, true));
+ // Verify the volume is deleted
+ Assert.assertFalse(volumeExist(volumeStr3));
+ }
+
+ @Test
+ public void testFailToDeleteRoot() throws IOException {
+ // rm root should always fail for OFS
+ Assert.assertFalse(fs.delete(new Path("/"), false));
+ Assert.assertFalse(fs.delete(new Path("/"), true));
+ }
+
}
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index b7323ac..4706183 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -467,6 +467,13 @@ public class BasicOzoneFileSystem extends FileSystem {
}
}
+ /**
+ * {@inheritDoc}
+ *
+ * OFS supports volume and bucket deletion, recursive or non-recursive.
+ * e.g. delete(new Path("/volume1"), true)
+ * But root deletion is explicitly disallowed for safety concerns.
+ */
@Override
public boolean delete(Path f, boolean recursive) throws IOException {
incrementCounter(Statistic.INVOCATION_DELETE);
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index bf7b124..171937e 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -307,6 +307,9 @@ public class BasicRootedOzoneClientAdapterImpl
boolean overWrite, boolean recursive) throws IOException {
incrementCounter(Statistic.OBJECTS_CREATED);
OFSPath ofsPath = new OFSPath(pathStr);
+ if (ofsPath.isRoot() || ofsPath.isVolume() || ofsPath.isBucket()) {
+ throw new IOException("Cannot create file under root or volume.");
+ }
String key = ofsPath.getKeyName();
try {
// Hadoop CopyCommands class always sets recursive to true
@@ -660,6 +663,10 @@ public class BasicRootedOzoneClientAdapterImpl
}
+ public ObjectStore getObjectStore() {
+ return objectStore;
+ }
+
@Override
public KeyProvider getKeyProvider() throws IOException {
return objectStore.getKeyProvider();
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 39aeef1..fd6df55 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -61,6 +62,8 @@ import static
org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
import static org.apache.hadoop.fs.ozone.Constants.OZONE_USER_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_EMPTY;
+import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_EMPTY;
/**
* The minimal Ozone Filesystem implementation.
@@ -84,6 +87,7 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
private String userName;
private Path workingDir;
private OzoneClientAdapter adapter;
+ private BasicRootedOzoneClientAdapterImpl adapterImpl;
private static final String URI_EXCEPTION_TEXT =
"URL should be one of the following formats: " +
@@ -149,6 +153,7 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
createAdapter(source,
omHostOrServiceId, omPort,
isolatedClassloader);
+ this.adapterImpl = (BasicRootedOzoneClientAdapterImpl) this.adapter;
try {
this.userName =
@@ -388,7 +393,7 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
}
private class DeleteIterator extends OzoneListingIterator {
- private boolean recursive;
+ final private boolean recursive;
private final OzoneBucket bucket;
private final BasicRootedOzoneClientAdapterImpl adapterImpl;
@@ -409,12 +414,12 @@ public class BasicRootedOzoneFileSystem extends
FileSystem {
}
@Override
- boolean processKeyPath(String keyPath) throws IOException {
+ boolean processKeyPath(String keyPath) {
if (keyPath.equals("")) {
LOG.trace("Skipping deleting root directory");
return true;
} else {
- LOG.trace("deleting key path:" + keyPath);
+ LOG.trace("Deleting: {}", keyPath);
boolean succeed = adapterImpl.deleteObject(this.bucket, keyPath);
// if recursive delete is requested ignore the return value of
// deleteObject and issue deletes for other keys.
@@ -457,19 +462,73 @@ public class BasicRootedOzoneFileSystem extends
FileSystem {
return false;
}
+ if (status == null) {
+ return false;
+ }
+
String key = pathToKey(f);
boolean result;
if (status.isDirectory()) {
LOG.debug("delete: Path is a directory: {}", f);
- key = addTrailingSlashIfNeeded(key);
-
- if (key.equals("/")) {
- LOG.warn("Cannot delete root directory.");
+ OFSPath ofsPath = new OFSPath(key);
+
+ // Handle rm root
+ if (ofsPath.isRoot()) {
+ // Intentionally drop support for rm root
+ // because it is too dangerous and doesn't provide much value
+ LOG.warn("delete: OFS does not support rm root. "
+ + "To wipe the cluster, please re-init OM instead.");
return false;
}
+ // Handle delete volume
+ if (ofsPath.isVolume()) {
+ String volumeName = ofsPath.getVolumeName();
+ if (recursive) {
+ // Delete all buckets first
+ OzoneVolume volume =
+ adapterImpl.getObjectStore().getVolume(volumeName);
+ Iterator<? extends OzoneBucket> it = volume.listBuckets("");
+ String prefixVolumePathStr = addTrailingSlashIfNeeded(f.toString());
+ while (it.hasNext()) {
+ OzoneBucket bucket = it.next();
+ String nextBucket = prefixVolumePathStr + bucket.getName();
+ delete(new Path(nextBucket), true);
+ }
+ }
+ try {
+ adapterImpl.getObjectStore().deleteVolume(volumeName);
+ return true;
+ } catch (OMException ex) {
+ // volume is not empty
+ if (ex.getResult() == VOLUME_NOT_EMPTY) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
result = innerDelete(f, recursive);
+
+ // Handle delete bucket
+ if (ofsPath.isBucket()) {
+ OzoneVolume volume =
+ adapterImpl.getObjectStore().getVolume(ofsPath.getVolumeName());
+ try {
+ volume.deleteBucket(ofsPath.getBucketName());
+ return result;
+ } catch (OMException ex) {
+ // bucket is not empty
+ if (ex.getResult() == BUCKET_NOT_EMPTY) {
+ throw new PathIsNotEmptyDirectoryException(f.toString());
+ } else {
+ throw ex;
+ }
+ }
+ }
+
} else {
LOG.debug("delete: Path is a file: {}", f);
result = adapter.deleteObject(key);
@@ -477,7 +536,7 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
if (result) {
// If this delete operation removes all files/directories from the
- // parent direcotry, then an empty parent directory must be created.
+ // parent directory, then an empty parent directory must be created.
createFakeParentDirectory(f);
}
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
index e970168..88f89fc 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
@@ -190,6 +190,17 @@ class OFSPath {
return this.getBucketName().isEmpty() && !this.getVolumeName().isEmpty();
}
+ /**
+ * If key name is empty but volume and bucket names are not, the given path
+ * it bucket.
+ * e.g. /volume1/bucket2
+ */
+ public boolean isBucket() {
+ return this.getKeyName().isEmpty() &&
+ !this.getBucketName().isEmpty() &&
+ !this.getVolumeName().isEmpty();
+ }
+
private static String md5Hex(String input) {
try {
MessageDigest md = MessageDigest.getInstance("MD5");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
