This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 845ceff4d6 HDDS-7603. Enable Erasure coding and encryption flag on
FileStatus for ofs (#4060)
845ceff4d6 is described below
commit 845ceff4d699a855be103a7271757a88455ed2ad
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Fri Dec 9 16:15:58 2022 -0700
HDDS-7603. Enable Erasure coding and encryption flag on FileStatus for ofs
(#4060)
---
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 42 ++++++++++++++++++++--
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 4 +--
.../fs/ozone/BasicRootedOzoneFileSystem.java | 39 ++++++++++----------
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 20 ++++++++++-
4 files changed, 80 insertions(+), 25 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index dbcb8218f3..cda335d6ba 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -22,6 +22,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -35,7 +36,6 @@ import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.fs.TrashPolicy;
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
@@ -89,10 +89,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
-import java.util.HashMap;
import java.util.Optional;
import java.util.Random;
import java.util.Set;
@@ -318,6 +318,44 @@ public class TestRootedOzoneFileSystem {
fs.delete(grandparent, true);
}
+ @Test
+ public void testListStatusWithIntermediateDirWithECEnabled()
+ throws Exception {
+ String key = "object-dir/object-name1";
+
+ // write some test data into bucket
+ try (OzoneOutputStream outputStream = objectStore.getVolume(volumeName).
+ getBucket(bucketName).createKey(key, 1,
+ new ECReplicationConfig("RS-3-2-1024"),
+ new HashMap<>())) {
+ outputStream.write(RandomUtils.nextBytes(1));
+ }
+
+ List<String> dirs = Arrays.asList(volumeName, bucketName, "object-dir",
+ "object-name1");
+ for (int size = 1; size <= dirs.size(); size++) {
+ String path = "/" + dirs.subList(0, size).stream()
+ .collect(Collectors.joining("/"));
+ Path parent = new Path(path);
+ // Wait until the filestatus is updated
+ if (!enabledFileSystemPaths) {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ fs.getFileStatus(parent);
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }, 1000, 120000);
+ }
+ FileStatus fileStatus = fs.getFileStatus(parent);
+ Assert.assertEquals((size == dirs.size() - 1 &&
+ !bucketLayout.isFileSystemOptimized()) || size == dirs.size(),
+ fileStatus.isErasureCoded());
+ }
+
+ }
+
@Test
public void testDeleteCreatesFakeParentDir() throws Exception {
// TODO: Request for comment.
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 40ed07e554..3819233a4c 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -1056,7 +1056,7 @@ public class BasicRootedOzoneClientAdapterImpl
return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L,
ozoneVolume.getCreationTime().getEpochSecond() * 1000, 0L,
FsPermission.getDirDefault().toShort(),
- owner, group, path, new BlockLocation[0], false, false
+ owner, group, null, new BlockLocation[0], false, false
);
}
@@ -1082,7 +1082,7 @@ public class BasicRootedOzoneClientAdapterImpl
return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L,
ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L,
FsPermission.getDirDefault().toShort(),
- owner, group, path, new BlockLocation[0],
+ owner, group, null, new BlockLocation[0],
!StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()),
ozoneBucket.getReplicationConfig() != null &&
ozoneBucket.getReplicationConfig().getReplicationType() ==
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index dc7c12ed9b..2a757a1e60 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -1323,28 +1323,27 @@ public class BasicRootedOzoneFileSystem extends
FileSystem {
return true;
}
- FileStatus convertFileStatus(FileStatusAdapter fileStatusAdapter) {
- Path symLink = null;
- try {
- fileStatusAdapter.getSymlink();
- } catch (Exception ex) {
- //NOOP: If not symlink symlink remains null.
- }
-
- FileStatus fileStatus = new FileStatus(
- fileStatusAdapter.getLength(),
- fileStatusAdapter.isDir(),
- fileStatusAdapter.getBlockReplication(),
- fileStatusAdapter.getBlocksize(),
- fileStatusAdapter.getModificationTime(),
- fileStatusAdapter.getAccessTime(),
- new FsPermission(fileStatusAdapter.getPermission()),
- fileStatusAdapter.getOwner(),
- fileStatusAdapter.getGroup(),
- symLink,
- fileStatusAdapter.getPath()
+ protected FileStatus constructFileStatus(
+ FileStatusAdapter fileStatusAdapter) {
+ return new FileStatus(fileStatusAdapter.getLength(),
+ fileStatusAdapter.isDir(),
+ fileStatusAdapter.getBlockReplication(),
+ fileStatusAdapter.getBlocksize(),
+ fileStatusAdapter.getModificationTime(),
+ fileStatusAdapter.getAccessTime(),
+ new FsPermission(fileStatusAdapter.getPermission()),
+ fileStatusAdapter.getOwner(),
+ fileStatusAdapter.getGroup(),
+ fileStatusAdapter.getSymlink(),
+ fileStatusAdapter.getPath(),
+ false,
+ fileStatusAdapter.isEncrypted(),
+ fileStatusAdapter.isErasureCoded()
);
+ }
+ FileStatus convertFileStatus(FileStatusAdapter fileStatusAdapter) {
+ FileStatus fileStatus = constructFileStatus(fileStatusAdapter);
BlockLocation[] blockLocations = fileStatusAdapter.getBlockLocations();
if (blockLocations == null || blockLocations.length == 0) {
return fileStatus;
diff --git
a/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
index 67aa705ab8..249b2767bd 100644
---
a/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
@@ -17,9 +17,27 @@
*/
package org.apache.hadoop.fs.ozone;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.permission.FsPermission;
+
/**
* Minimal Rooted Ozone File System compatible with Hadoop 2.x.
*/
public class RootedOzoneFileSystem extends BasicRootedOzoneFileSystem {
-
+ @Override
+ protected FileStatus constructFileStatus(
+ FileStatusAdapter fileStatusAdapter) {
+ return new FileStatus(fileStatusAdapter.getLength(),
+ fileStatusAdapter.isDir(),
+ fileStatusAdapter.getBlockReplication(),
+ fileStatusAdapter.getBlocksize(),
+ fileStatusAdapter.getModificationTime(),
+ fileStatusAdapter.getAccessTime(),
+ new FsPermission(fileStatusAdapter.getPermission()),
+ fileStatusAdapter.getOwner(),
+ fileStatusAdapter.getGroup(),
+ fileStatusAdapter.getSymlink(),
+ fileStatusAdapter.getPath()
+ );
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
