This is an automated email from the ASF dual-hosted git repository.
jojochuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6fd35fbf92f HDDS-14043. Fix ls -e UnsupportedOperationException on
ofs/o3fs (#10209)
6fd35fbf92f is described below
commit 6fd35fbf92f7923776af97bb55022e485fe32365
Author: Chi-Hsuan Huang <[email protected]>
AuthorDate: Thu Jul 16 03:35:43 2026 +0800
HDDS-14043. Fix ls -e UnsupportedOperationException on ofs/o3fs (#10209)
Generated-by: Copilot Autofix powered by AI
<[email protected]>
---
.../fs/ozone/AbstractOzoneFileSystemTest.java | 44 +++++++++++
.../ozone/AbstractRootedOzoneFileSystemTest.java | 51 +++++++++++++
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 20 ++++-
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 86 ++++++++++++++++++++++
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 47 ++++++++----
.../fs/ozone/BasicRootedOzoneFileSystem.java | 34 +++++++--
.../apache/hadoop/fs/ozone/FileStatusAdapter.java | 20 +++++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 ++
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 ++
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 ++
.../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 ++
11 files changed, 311 insertions(+), 23 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
index 32d97295917..638bb93e813 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
@@ -73,6 +73,7 @@
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
@@ -403,6 +404,49 @@ public void testCreateKeyWithECReplicationConfig() throws
Exception {
createKeyWithECReplicationConfig(root, cluster.getConf());
}
+ @Test
+ void testContentSummaryErasureCodingPolicy() throws Exception {
+ String ratisKey = "ratis-ec-policy-key";
+ String ecKey = "ec-policy-key";
+ ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k");
+ Path parentDir = new Path(OZONE_URI_DELIMITER, "ec-policy-mixed-o3fs");
+ Path ratisFile = new Path(parentDir, ratisKey);
+ Path ecFile = new Path(parentDir, ecKey);
+
+ fs.mkdirs(parentDir);
+ String ratisRelKey = "ec-policy-mixed-o3fs/" + ratisKey;
+ String ecRelKey = "ec-policy-mixed-o3fs/" + ecKey;
+ TestDataUtil.createKey(ozoneBucket, ratisRelKey,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE),
+ new byte[]{0});
+ TestDataUtil.createKey(ozoneBucket, ecRelKey, ecConfig,
+ new byte[]{0});
+
+ try {
+ assertEquals("",
+ fs.getContentSummary(ROOT).getErasureCodingPolicy());
+ assertEquals("Replicated",
+ fs.getContentSummary(ratisFile).getErasureCodingPolicy());
+ assertEquals(ecConfig.getReplication(),
+ fs.getContentSummary(ecFile).getErasureCodingPolicy());
+ assertEquals("",
+ fs.getContentSummary(parentDir).getErasureCodingPolicy());
+ } finally {
+ fs.delete(parentDir, true);
+ }
+ }
+
+ @Test
+ void testLsDashEDoesNotThrow() throws Exception {
+ FsShell shell = new FsShell(fs.getConf());
+ try {
+ int exitCode = shell.run(new String[]{"-ls", "-R", "-e", fsRoot});
+ assertEquals(0, exitCode);
+ } finally {
+ shell.close();
+ }
+ }
+
@Test
public void testDeleteCreatesFakeParentDir() throws Exception {
deleteCreatesFakeParentDir(ROOT);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
index 44d9baec556..aa6431fc56c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
@@ -77,6 +77,7 @@
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.StreamCapabilities;
@@ -1859,6 +1860,56 @@ void testCreateAndCheckECFileDiskUsage() throws
Exception {
fs.delete(filePath, true);
}
+ @Test
+ void testContentSummaryErasureCodingPolicy() throws Exception {
+ String ratisKey = "ratis-ec-policy-key";
+ String ecKey = "ec-policy-key";
+ ECReplicationConfig ecConfig = new ECReplicationConfig("RS-3-2-1024k");
+ Path parentDir = new Path(bucketPath, "ec-policy-mixed");
+ Path ratisFile = new Path(parentDir, ratisKey);
+ Path ecFile = new Path(parentDir, ecKey);
+
+ fs.mkdirs(parentDir);
+ OzoneBucket bucket =
objectStore.getVolume(volumeName).getBucket(bucketName);
+ String ratisRelKey = "ec-policy-mixed/" + ratisKey;
+ String ecRelKey = "ec-policy-mixed/" + ecKey;
+ TestDataUtil.createKey(bucket, ratisRelKey,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE),
+ new byte[]{0});
+ TestDataUtil.createKey(bucket, ecRelKey, ecConfig,
+ new byte[]{0});
+
+ try {
+ assertEquals("",
+ fs.getContentSummary(new Path(OZONE_URI_DELIMITER))
+ .getErasureCodingPolicy());
+ assertEquals("",
+ fs.getContentSummary(volumePath).getErasureCodingPolicy());
+ assertEquals("",
+ fs.getContentSummary(bucketPath).getErasureCodingPolicy());
+ assertEquals("Replicated",
+ fs.getContentSummary(ratisFile).getErasureCodingPolicy());
+ assertEquals(ecConfig.getReplication(),
+ fs.getContentSummary(ecFile).getErasureCodingPolicy());
+ assertEquals("",
+ fs.getContentSummary(parentDir).getErasureCodingPolicy());
+ } finally {
+ fs.delete(parentDir, true);
+ }
+ }
+
+ @Test
+ void testLsDashEDoesNotThrow() throws Exception {
+ FsShell shell = new FsShell(conf);
+ try {
+ int exitCode = shell.run(new String[]{"-ls", "-R", "-e",
+ OZONE_URI_DELIMITER + volumeName + OZONE_URI_DELIMITER +
bucketName});
+ assertEquals(0, exitCode);
+ } finally {
+ shell.close();
+ }
+ }
+
@Test
void testCreateAndCheckRatisFileDiskUsage() throws Exception {
String key = "ratiskeytest";
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index f8557b61e46..d9033054ead 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -537,6 +537,13 @@ private FileStatusAdapter
toFileStatusAdapter(OzoneFileStatus status,
OmKeyInfo keyInfo = status.getKeyInfo();
short replication = (short) keyInfo.getReplicationConfig()
.getRequiredNodes();
+ boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo);
+ String ecPolicy;
+ if (isEc) {
+ ecPolicy = keyInfo.getReplicationConfig().getReplication();
+ } else {
+ ecPolicy = status.isFile() ? "Replicated" : "";
+ }
return new FileStatusAdapter(
keyInfo.getDataSize(),
keyInfo.getReplicatedSize(),
@@ -553,7 +560,8 @@ private FileStatusAdapter
toFileStatusAdapter(OzoneFileStatus status,
null,
getBlockLocations(status),
OzoneClientUtils.isKeyEncrypted(keyInfo),
- OzoneClientUtils.isKeyErasureCode(keyInfo)
+ isEc,
+ ecPolicy
);
}
@@ -562,6 +570,13 @@ private FileStatusAdapter
toFileStatusAdapter(OzoneFileStatusLight status,
BasicOmKeyInfo keyInfo = status.getKeyInfo();
short replication = (short) keyInfo.getReplicationConfig()
.getRequiredNodes();
+ boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo);
+ String ecPolicy;
+ if (isEc) {
+ ecPolicy = keyInfo.getReplicationConfig().getReplication();
+ } else {
+ ecPolicy = status.isFile() ? "Replicated" : "";
+ }
return new FileStatusAdapter(
keyInfo.getDataSize(),
keyInfo.getReplicatedSize(),
@@ -578,7 +593,8 @@ private FileStatusAdapter
toFileStatusAdapter(OzoneFileStatusLight status,
null,
getBlockLocations(null),
keyInfo.isEncrypted(),
- OzoneClientUtils.isKeyErasureCode(keyInfo)
+ isEc,
+ ecPolicy
);
}
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index 8e0f4e8a5dd..4f0cc8b60bd 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -51,6 +51,7 @@
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -865,6 +866,91 @@ public BlockLocation[] getFileBlockLocations(FileStatus
fileStatus,
}
}
+ @Override
+ public ContentSummary getContentSummary(Path f) throws IOException {
+ Path qualifiedPath = f.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+ FileStatusAdapter status;
+ try {
+ status = adapter.getFileStatus(key, uri, qualifiedPath, getUsername());
+ } catch (OMException ex) {
+ if (ex.getResult().equals(OMException.ResultCodes.KEY_NOT_FOUND)) {
+ throw new FileNotFoundException("File not found. path:" + f);
+ }
+ throw ex;
+ }
+
+ if (status.isFile()) {
+ long length = status.getLength();
+ long spaceConsumed = status.getDiskConsumed();
+ ContentSummary.Builder builder = new
ContentSummary.Builder().length(length).
+ fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed);
+ applyEcPolicy(builder, status.getErasureCodingPolicy());
+ return builder.build();
+ }
+
+ long[] summary = {0, 0, 0, 1};
+ for (FileStatusAdapter s : listStatusAdapter(f)) {
+ long length = s.getLength();
+ long spaceConsumed = s.getDiskConsumed();
+ ContentSummary c;
+ if (s.isDir()) {
+ c = getContentSummary(s.getPath());
+ } else {
+ ContentSummary.Builder childBuilder = new
ContentSummary.Builder().length(length).
+ fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed);
+ applyEcPolicy(childBuilder, s.getErasureCodingPolicy());
+ c = childBuilder.build();
+ }
+
+ summary[0] += c.getLength();
+ summary[1] += c.getSpaceConsumed();
+ summary[2] += c.getFileCount();
+ summary[3] += c.getDirectoryCount();
+ }
+
+ ContentSummary.Builder builder = new
ContentSummary.Builder().length(summary[0]).
+ fileCount(summary[2]).directoryCount(summary[3]).
+ spaceConsumed(summary[1]);
+ applyEcPolicy(builder, status.getErasureCodingPolicy());
+ return builder.build();
+ }
+
+ /**
+ * Apply the erasure coding policy on the {@link ContentSummary.Builder}.
+ * Default implementation is a no-op so that this class can compile and run
+ * against Hadoop 2, where {@code ContentSummary.Builder.erasureCodingPolicy}
+ * does not exist. The Hadoop 3 subclass overrides this to set the policy.
+ */
+ protected void applyEcPolicy(ContentSummary.Builder builder, String
ecPolicy) {
+ }
+
+ private List<FileStatusAdapter> listStatusAdapter(Path f) throws IOException
{
+ int numEntries = listingPageSize;
+ LinkedList<FileStatusAdapter> statuses = new LinkedList<>();
+ List<FileStatusAdapter> tmpStatusList;
+ String startKey = "";
+ int entriesAdded;
+ do {
+ tmpStatusList = adapter.listStatus(pathToKey(f), false, startKey,
+ numEntries, uri, workingDir, getUsername(), true);
+ entriesAdded = 0;
+ if (!tmpStatusList.isEmpty()) {
+ if (startKey.isEmpty() || !statuses.getLast().getPath().toString()
+ .equals(tmpStatusList.get(0).getPath().toString())) {
+ statuses.addAll(tmpStatusList);
+ entriesAdded += tmpStatusList.size();
+ } else {
+ statuses.addAll(tmpStatusList.subList(1, tmpStatusList.size()));
+ entriesAdded += tmpStatusList.size() - 1;
+ }
+ startKey = pathToKey(statuses.getLast().getPath());
+ }
+ } while (entriesAdded > 0);
+
+ return statuses;
+ }
+
@Override
public short getDefaultReplication() {
return adapter.getDefaultReplication();
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index c502f9096c5..0b11ab95de5 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -1035,6 +1035,13 @@ private FileStatusAdapter
toFileStatusAdapter(OzoneFileStatus status,
OmKeyInfo keyInfo = status.getKeyInfo();
short replication = (short) keyInfo.getReplicationConfig()
.getRequiredNodes();
+ boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo);
+ String ecPolicy;
+ if (isEc) {
+ ecPolicy = keyInfo.getReplicationConfig().getReplication();
+ } else {
+ ecPolicy = status.isFile() ? "Replicated" : "";
+ }
return new FileStatusAdapter(
keyInfo.getDataSize(),
keyInfo.getReplicatedSize(),
@@ -1051,7 +1058,8 @@ private FileStatusAdapter
toFileStatusAdapter(OzoneFileStatus status,
null,
getBlockLocations(status),
OzoneClientUtils.isKeyEncrypted(keyInfo),
- OzoneClientUtils.isKeyErasureCode(keyInfo)
+ isEc,
+ ecPolicy
);
}
@@ -1060,6 +1068,13 @@ private FileStatusAdapter
toFileStatusAdapter(OzoneFileStatusLight status,
BasicOmKeyInfo keyInfo = status.getKeyInfo();
short replication = (short) keyInfo.getReplicationConfig()
.getRequiredNodes();
+ boolean isEc = OzoneClientUtils.isKeyErasureCode(keyInfo);
+ String ecPolicy;
+ if (isEc) {
+ ecPolicy = keyInfo.getReplicationConfig().getReplication();
+ } else {
+ ecPolicy = status.isFile() ? "Replicated" : "";
+ }
return new FileStatusAdapter(
keyInfo.getDataSize(),
keyInfo.getReplicatedSize(),
@@ -1076,7 +1091,8 @@ private FileStatusAdapter
toFileStatusAdapter(OzoneFileStatusLight status,
null,
getBlockLocations(null),
keyInfo.isEncrypted(),
- OzoneClientUtils.isKeyErasureCode(keyInfo)
+ isEc,
+ ecPolicy
);
}
@@ -1169,7 +1185,7 @@ private static FileStatusAdapter
getFileStatusAdapterForVolume(
return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L,
ozoneVolume.getCreationTime().getEpochSecond() * 1000, 0L,
FsPermission.getDirDefault().toShort(),
- owner, group, null, new BlockLocation[0], false, false
+ owner, group, null, new BlockLocation[0], false, false, ""
);
}
@@ -1192,14 +1208,16 @@ private static FileStatusAdapter
getFileStatusAdapterForBucket(OzoneBucket ozone
UserGroupInformation ugi =
UserGroupInformation.createRemoteUser(ozoneBucket.getOwner());
String owner = ugi.getShortUserName();
String group = getGroupName(ugi);
+ ReplicationConfig rc = ozoneBucket.getReplicationConfig();
+ boolean isEc = rc != null && rc.getReplicationType() ==
HddsProtos.ReplicationType.EC;
+ String ecPolicy = isEc ? rc.getReplication() : "";
return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L,
ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L,
FsPermission.getDirDefault().toShort(),
owner, group, null, new BlockLocation[0],
!StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()),
- ozoneBucket.getReplicationConfig() != null &&
- ozoneBucket.getReplicationConfig().getReplicationType() ==
- HddsProtos.ReplicationType.EC);
+ isEc,
+ ecPolicy);
}
/**
@@ -1225,6 +1243,9 @@ private static FileStatusAdapter
getFileStatusAdapterForBucketSnapshot(
ozoneSnapshot.getName(), pathStr);
}
Path path = new Path(pathStr);
+ ReplicationConfig rc = ozoneBucket.getReplicationConfig();
+ boolean isEc = rc != null && rc.getReplicationType() ==
HddsProtos.ReplicationType.EC;
+ String ecPolicy = isEc ? rc.getReplication() : "";
return new FileStatusAdapter(
ozoneSnapshot.getReferencedSize(),
ozoneSnapshot.getReferencedReplicatedSize(),
@@ -1233,9 +1254,8 @@ private static FileStatusAdapter
getFileStatusAdapterForBucketSnapshot(
FsPermission.getDirDefault().toShort(),
owner, group, null, new BlockLocation[0],
!StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()),
- ozoneBucket.getReplicationConfig() != null &&
- ozoneBucket.getReplicationConfig().getReplicationType() ==
- HddsProtos.ReplicationType.EC);
+ isEc,
+ ecPolicy);
}
/**
@@ -1264,14 +1284,15 @@ private static FileStatusAdapter
getFileStatusAdapterWithSnapshotIndicator(
ozoneBucket.getName(), pathStr);
}
Path path = new Path(pathStr);
+ boolean isEc = false;
+ String ecPolicy = "";
return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L,
ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L,
FsPermission.getDirDefault().toShort(),
owner, group, null, new BlockLocation[0],
!StringUtils.isEmpty(ozoneBucket.getEncryptionKeyName()),
- ozoneBucket.getReplicationConfig() != null &&
- ozoneBucket.getReplicationConfig().getReplicationType() ==
- HddsProtos.ReplicationType.EC);
+ isEc,
+ ecPolicy);
}
/**
@@ -1286,7 +1307,7 @@ private static FileStatusAdapter
getFileStatusAdapterForRoot(URI uri) {
return new FileStatusAdapter(0L, 0L, path, true, (short)0, 0L,
System.currentTimeMillis(), 0L,
FsPermission.getDirDefault().toShort(),
- null, null, null, new BlockLocation[0], false, false);
+ null, null, null, new BlockLocation[0], false, false, "");
}
@Override
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 67f09313d0e..b61dca450cb 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -1575,18 +1575,25 @@ private ContentSummary getContentSummaryInSpan(Path f)
throws IOException {
long length = status.getLength();
long spaceConsumed = status.getDiskConsumed();
- return new ContentSummary.Builder().length(length).
- fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed).build();
+ ContentSummary.Builder builder = new
ContentSummary.Builder().length(length).
+ fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed);
+ applyEcPolicy(builder, status.getErasureCodingPolicy());
+ return builder.build();
}
// f is a directory
long[] summary = {0, 0, 0, 1};
- int i = 0;
for (FileStatusAdapter s : listStatusAdapter(f, true)) {
long length = s.getLength();
long spaceConsumed = s.getDiskConsumed();
- ContentSummary c = s.isDir() ? getContentSummary(s.getPath()) :
- new ContentSummary.Builder().length(length).
- fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed).build();
+ ContentSummary c;
+ if (s.isDir()) {
+ c = getContentSummary(s.getPath());
+ } else {
+ ContentSummary.Builder childBuilder = new
ContentSummary.Builder().length(length).
+ fileCount(1).directoryCount(0).spaceConsumed(spaceConsumed);
+ applyEcPolicy(childBuilder, s.getErasureCodingPolicy());
+ c = childBuilder.build();
+ }
summary[0] += c.getLength();
summary[1] += c.getSpaceConsumed();
@@ -1594,9 +1601,20 @@ private ContentSummary getContentSummaryInSpan(Path f)
throws IOException {
summary[3] += c.getDirectoryCount();
}
- return new ContentSummary.Builder().length(summary[0]).
+ ContentSummary.Builder builder = new
ContentSummary.Builder().length(summary[0]).
fileCount(summary[2]).directoryCount(summary[3]).
- spaceConsumed(summary[1]).build();
+ spaceConsumed(summary[1]);
+ applyEcPolicy(builder, status.getErasureCodingPolicy());
+ return builder.build();
+ }
+
+ /**
+ * Apply the erasure coding policy on the {@link ContentSummary.Builder}.
+ * Default implementation is a no-op so that this class can compile and run
+ * against Hadoop 2, where {@code ContentSummary.Builder.erasureCodingPolicy}
+ * does not exist. The Hadoop 3 subclass overrides this to set the policy.
+ */
+ protected void applyEcPolicy(ContentSummary.Builder builder, String
ecPolicy) {
}
@Override
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java
index eba24f88bb5..232a1e31b53 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/FileStatusAdapter.java
@@ -53,6 +53,8 @@ public final class FileStatusAdapter {
private final boolean isErasureCoded;
+ private final String erasureCodingPolicy;
+
@SuppressWarnings("checkstyle:ParameterNumber")
public FileStatusAdapter(long length, long diskConsumed, Path path,
boolean isdir, short blockReplication, long blocksize,
@@ -60,6 +62,18 @@ public FileStatusAdapter(long length, long diskConsumed,
Path path,
String owner, String group, Path symlink,
BlockLocation[] locations, boolean isEncrypted,
boolean isErasureCoded) {
+ this(length, diskConsumed, path, isdir, blockReplication, blocksize,
+ modificationTime, accessTime, permission, owner, group, symlink,
+ locations, isEncrypted, isErasureCoded, null);
+ }
+
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ public FileStatusAdapter(long length, long diskConsumed, Path path,
+ boolean isdir, short blockReplication, long blocksize,
+ long modificationTime, long accessTime, short permission,
+ String owner, String group, Path symlink,
+ BlockLocation[] locations, boolean isEncrypted,
+ boolean isErasureCoded, String erasureCodingPolicy) {
this.length = length;
this.diskConsumed = diskConsumed;
this.path = path;
@@ -75,6 +89,7 @@ public FileStatusAdapter(long length, long diskConsumed, Path
path,
this.blockLocations = new ArrayList<>(Arrays.asList(locations));
this.isEncrypted = isEncrypted;
this.isErasureCoded = isErasureCoded;
+ this.erasureCodingPolicy = erasureCodingPolicy;
}
public Path getPath() {
@@ -137,6 +152,10 @@ public boolean isErasureCoded() {
return isErasureCoded;
}
+ public String getErasureCodingPolicy() {
+ return erasureCodingPolicy;
+ }
+
public BlockLocation[] getBlockLocations() {
return blockLocations.toArray(new BlockLocation[0]);
}
@@ -159,6 +178,7 @@ public String toString() {
.append("; group=").append(group)
.append("; permission=").append(permission)
.append("; isSymlink=").append(getSymlink())
+ .append("; erasureCodingPolicy=").append(erasureCodingPolicy)
.append('}');
return sb.toString();
diff --git
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 5136c734307..b23f0fb5087 100644
---
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -25,6 +25,7 @@
import java.util.List;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LeaseRecoverable;
import org.apache.hadoop.fs.Path;
@@ -183,4 +184,11 @@ public boolean setSafeMode(SafeModeAction action, boolean
isChecked)
throws IOException {
return setSafeModeUtil(action, isChecked);
}
+
+ @Override
+ protected void applyEcPolicy(ContentSummary.Builder builder, String
ecPolicy) {
+ if (ecPolicy != null) {
+ builder.erasureCodingPolicy(ecPolicy);
+ }
+ }
}
diff --git
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
index 6774b658856..0031e57d31e 100644
---
a/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
@@ -25,6 +25,7 @@
import java.util.List;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LeaseRecoverable;
import org.apache.hadoop.fs.Path;
@@ -188,4 +189,11 @@ public boolean setSafeMode(SafeModeAction action, boolean
isChecked)
throws IOException {
return setSafeModeUtil(action, isChecked);
}
+
+ @Override
+ protected void applyEcPolicy(ContentSummary.Builder builder, String
ecPolicy) {
+ if (ecPolicy != null) {
+ builder.erasureCodingPolicy(ecPolicy);
+ }
+ }
}
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 5136c734307..b23f0fb5087 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -25,6 +25,7 @@
import java.util.List;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LeaseRecoverable;
import org.apache.hadoop.fs.Path;
@@ -183,4 +184,11 @@ public boolean setSafeMode(SafeModeAction action, boolean
isChecked)
throws IOException {
return setSafeModeUtil(action, isChecked);
}
+
+ @Override
+ protected void applyEcPolicy(ContentSummary.Builder builder, String
ecPolicy) {
+ if (ecPolicy != null) {
+ builder.erasureCodingPolicy(ecPolicy);
+ }
+ }
}
diff --git
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
index 74c4a30e991..38dc72a7727 100644
---
a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/RootedOzoneFileSystem.java
@@ -25,6 +25,7 @@
import java.util.List;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderTokenIssuer;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LeaseRecoverable;
import org.apache.hadoop.fs.Path;
@@ -195,4 +196,11 @@ public boolean setSafeMode(SafeModeAction action, boolean
isChecked)
throws IOException {
return setSafeModeUtil(action, isChecked);
}
+
+ @Override
+ protected void applyEcPolicy(ContentSummary.Builder builder, String
ecPolicy) {
+ if (ecPolicy != null) {
+ builder.erasureCodingPolicy(ecPolicy);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
