This is an automated email from the ASF dual-hosted git repository.
ritesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6faf0023b0 HDDS-7715. Implement a lightweight listStatus API (#5275)
6faf0023b0 is described below
commit 6faf0023b057dc2cc51bdbf288a43e756fb42b4b
Author: tanvipenumudy <[email protected]>
AuthorDate: Fri Oct 27 12:14:22 2023 +0530
HDDS-7715. Implement a lightweight listStatus API (#5275)
---
.../apache/hadoop/ozone/client/OzoneBucket.java | 21 +--
.../ozone/client/protocol/ClientProtocol.java | 21 ++-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 11 ++
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../apache/hadoop/ozone/om/IOmMetadataReader.java | 19 +++
.../hadoop/ozone/om/helpers/BasicOmKeyInfo.java | 24 ++++
.../ozone/om/helpers/OzoneFileStatusLight.java | 157 +++++++++++++++++++++
.../ozone/om/protocol/OzoneManagerProtocol.java | 18 +++
...OzoneManagerProtocolClientSideTranslatorPB.java | 40 ++++++
.../src/main/proto/OmClientProtocol.proto | 18 +++
.../apache/hadoop/ozone/om/OmMetadataReader.java | 13 ++
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 14 ++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 13 ++
.../protocolPB/OzoneManagerRequestHandler.java | 34 +++++
.../hadoop/ozone/client/ClientProtocolStub.java | 8 ++
15 files changed, 402 insertions(+), 10 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index bc8dcdd0e5..c4c68c00fd 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -38,10 +38,12 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.WithMetadata;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -1226,8 +1228,9 @@ public class OzoneBucket extends WithMetadata {
// Elements in statuses must be sorted after startKey,
// which means they come after the keyPrefix.
- List<OzoneFileStatus> statuses = proxy.listStatus(volumeName, name,
- delimiterKeyPrefix, false, startKey, listCacheSize);
+ List<OzoneFileStatusLight> statuses =
+ proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false,
+ startKey, listCacheSize, false);
if (addedKeyPrefix) {
// previous round already include the startKey, so remove it
@@ -1246,10 +1249,10 @@ public class OzoneBucket extends WithMetadata {
}
private List<OzoneKey> buildOzoneKeysFromFileStatus(
- List<OzoneFileStatus> statuses) {
+ List<OzoneFileStatusLight> statuses) {
return statuses.stream()
.map(status -> {
- OmKeyInfo keyInfo = status.getKeyInfo();
+ BasicOmKeyInfo keyInfo = status.getKeyInfo();
String keyName = keyInfo.getKeyName();
if (status.isDirectory()) {
// add trailing slash to represent directory
@@ -1497,8 +1500,8 @@ public class OzoneBucket extends WithMetadata {
startKey = startKey == null ? "" : startKey;
// 1. Get immediate children of keyPrefix, starting with startKey
- List<OzoneFileStatus> statuses = proxy.listStatus(volumeName, name,
- keyPrefix, false, startKey, listCacheSize, true);
+ List<OzoneFileStatusLight> statuses = proxy.listStatusLight(volumeName,
+ name, keyPrefix, false, startKey, listCacheSize, true);
boolean reachedLimitCacheSize = statuses.size() == listCacheSize;
// 2. Special case: ListKey expects keyPrefix element should present in
@@ -1516,8 +1519,8 @@ public class OzoneBucket extends WithMetadata {
// 4. Iterating over the resultStatuses list and add each key to the
// resultList.
for (int indx = 0; indx < statuses.size(); indx++) {
- OzoneFileStatus status = statuses.get(indx);
- OmKeyInfo keyInfo = status.getKeyInfo();
+ OzoneFileStatusLight status = statuses.get(indx);
+ BasicOmKeyInfo keyInfo = status.getKeyInfo();
String keyName = keyInfo.getKeyName();
OzoneKey ozoneKey;
@@ -1559,7 +1562,7 @@ public class OzoneBucket extends WithMetadata {
}
private void removeStartKeyIfExistsInStatusList(String startKey,
- List<OzoneFileStatus> statuses) {
+ List<OzoneFileStatusLight> statuses) {
if (!statuses.isEmpty()) {
String firstElement = statuses.get(0).getKeyInfo().getKeyName();
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 3d51341ec0..b45a3209f4 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
@@ -881,7 +882,6 @@ public interface ClientProtocol {
String keyName, boolean recursive, String startKey, long numEntries)
throws IOException;
-
/**
* List the status for a file or a directory and its contents.
*
@@ -901,6 +901,25 @@ public interface ClientProtocol {
String keyName, boolean recursive, String startKey,
long numEntries, boolean allowPartialPrefixes) throws IOException;
+ /**
+ * Lightweight listStatus API.
+ *
+ * @param volumeName Volume name
+ * @param bucketName Bucket name
+ * @param keyName Absolute path of the entry to be listed
+ * @param recursive For a directory if true all the descendants of a
+ * particular directory are listed
+ * @param startKey Key from which listing needs to start. If startKey
exists
+ * its status is included in the final list.
+ * @param numEntries Number of entries to list from the start key
+ * @param allowPartialPrefixes if partial prefixes should be allowed,
+ * this is needed in context of ListKeys
+ * @return list of file status
+ */
+ List<OzoneFileStatusLight> listStatusLight(String volumeName,
+ String bucketName, String keyName, boolean recursive, String startKey,
+ long numEntries, boolean allowPartialPrefixes) throws IOException;
+
/**
* Add acl for Ozone object. Return true if acl is added successfully else
* false.
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index d3a3106934..fc91335771 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -131,6 +131,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
@@ -2074,6 +2075,16 @@ public class RpcClient implements ClientProtocol {
allowPartialPrefixes);
}
+ @Override
+ public List<OzoneFileStatusLight> listStatusLight(String volumeName,
+ String bucketName, String keyName, boolean recursive, String startKey,
+ long numEntries, boolean allowPartialPrefixes) throws IOException {
+ OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName);
+ return ozoneManagerClient
+ .listStatusLight(keyArgs, recursive, startKey, numEntries,
+ allowPartialPrefixes);
+ }
+
/**
* Add acl for Ozone object. Return true if acl is added successfully else
* false.
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index a4641148d3..c7cb996fdc 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -248,6 +248,7 @@ public final class OmUtils {
case GetFileStatus:
case LookupFile:
case ListStatus:
+ case ListStatusLight:
case GetAcl:
case DBUpdates:
case ListMultipartUploads:
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
index c0a920aca9..7d4e769365 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import java.io.IOException;
@@ -64,6 +65,24 @@ public interface IOmMetadataReader {
boolean allowPartialPrefixes)
throws IOException;
+ /**
+ * Lightweight listStatus API.
+ *
+ * @param args Key args
+ * @param recursive For a directory if true all the descendants of a
+ * particular directory are listed
+ * @param startKey Key from which listing needs to start. If startKey
exists
+ * its status is included in the final list.
+ * @param numEntries Number of entries to list from the start key
+ * @param allowPartialPrefixes if partial prefixes should be allowed,
+ * this is needed in context of ListKeys
+ * @return list of file status
+ */
+ List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args, boolean recursive,
+ String startKey, long numEntries,
+ boolean allowPartialPrefixes)
+ throws IOException;
+
default List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
String startKey, long numEntries)
throws IOException {
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java
index 2d7d859832..9c9a502777 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java
@@ -186,6 +186,30 @@ public class BasicOmKeyInfo {
return builder.build();
}
+ public static BasicOmKeyInfo getFromProtobuf(String volumeName,
+ String bucketName, BasicKeyInfo basicKeyInfo) throws IOException {
+ if (basicKeyInfo == null) {
+ return null;
+ }
+
+ String keyName = basicKeyInfo.getKeyName();
+
+ Builder builder = new Builder()
+ .setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setDataSize(basicKeyInfo.getDataSize())
+ .setCreationTime(basicKeyInfo.getCreationTime())
+ .setModificationTime(basicKeyInfo.getModificationTime())
+ .setReplicationConfig(ReplicationConfig.fromProto(
+ basicKeyInfo.getType(),
+ basicKeyInfo.getFactor(),
+ basicKeyInfo.getEcReplicationConfig()))
+ .setIsFile(!keyName.endsWith("/"));
+
+ return builder.build();
+ }
+
public boolean equals(Object o) {
if (this == o) {
return true;
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatusLight.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatusLight.java
new file mode 100644
index 0000000000..8956905bf7
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFileStatusLight.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProtoLight;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
+
+/**
+ * Lightweight OzoneFileStatus class.
+ */
+public class OzoneFileStatusLight {
+
+ private BasicOmKeyInfo keyInfo;
+
+ private boolean isDirectory;
+
+ private long blockSize;
+
+ public OzoneFileStatusLight() {
+ isDirectory = true;
+ }
+
+ public OzoneFileStatusLight(BasicOmKeyInfo keyInfo,
+ long blockSize, boolean isDirectory) {
+ this.keyInfo = keyInfo;
+ this.isDirectory = isDirectory;
+ this.blockSize = blockSize;
+ }
+
+ public BasicOmKeyInfo getKeyInfo() {
+ return keyInfo;
+ }
+
+ public long getBlockSize() {
+ return blockSize;
+ }
+
+ public String getTrimmedName() {
+ String keyName = keyInfo.getKeyName();
+ if (keyName.endsWith(OZONE_URI_DELIMITER)) {
+ return keyName.substring(0, keyName.length() - 1);
+ } else {
+ return keyName;
+ }
+ }
+
+ public String getPath() {
+ if (keyInfo == null) {
+ return OZONE_URI_DELIMITER;
+ } else {
+ String path = OZONE_URI_DELIMITER + keyInfo.getKeyName();
+ if (path.endsWith(OZONE_URI_DELIMITER)) {
+ return path.substring(0, path.length() - 1);
+ } else {
+ return path;
+ }
+ }
+ }
+
+ public boolean isDirectory() {
+ if (keyInfo == null) {
+ return true;
+ }
+ return isDirectory;
+ }
+
+ public boolean isFile() {
+ return !isDirectory();
+ }
+
+ public OzoneFileStatusProtoLight getProtobuf() {
+ OzoneFileStatusProtoLight.Builder builder =
+ OzoneFileStatusProtoLight.newBuilder()
+ .setBlockSize(blockSize)
+ .setIsDirectory(isDirectory);
+ //key info can be null for the fake root entry.
+ if (keyInfo != null) {
+ builder.setBasicKeyInfo(keyInfo.getProtobuf())
+ .setVolumeName(keyInfo.getVolumeName())
+ .setBucketName(keyInfo.getBucketName());
+ }
+ return builder.build();
+ }
+
+ public static OzoneFileStatusLight getFromProtobuf(
+ OzoneFileStatusProtoLight status) throws IOException {
+ return new OzoneFileStatusLight(
+ BasicOmKeyInfo.getFromProtobuf(status.getVolumeName(),
+ status.getBucketName(), status.getBasicKeyInfo()),
+ status.getBlockSize(),
+ status.getIsDirectory());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof OzoneFileStatusLight)) {
+ return false;
+ }
+ OzoneFileStatusLight that = (OzoneFileStatusLight) o;
+ return isDirectory == that.isDirectory &&
+ blockSize == that.blockSize &&
+ getTrimmedName().equals(that.getTrimmedName());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getTrimmedName());
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getClass().getSimpleName());
+ sb.append("{");
+ if (keyInfo == null) {
+ sb.append("<root>");
+ } else {
+ sb.append(getTrimmedName());
+ if (isDirectory) {
+ sb.append(" (dir)");
+ }
+ }
+ sb.append("}");
+ return sb.toString();
+ }
+
+ public static OzoneFileStatusLight fromOzoneFileStatus(
+ OzoneFileStatus ozoneFileStatus) {
+ BasicOmKeyInfo basicKeyInfo =
+ BasicOmKeyInfo.fromOmKeyInfo(ozoneFileStatus.getKeyInfo());
+ return new OzoneFileStatusLight(basicKeyInfo,
+ ozoneFileStatus.getBlockSize(), ozoneFileStatus.isDirectory());
+ }
+}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index eddba4f96f..7d710ae871 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.ozone.om.helpers.OmTenantArgs;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
@@ -912,6 +913,23 @@ public interface OzoneManagerProtocol
boolean allowPartialPrefixes)
throws IOException;
+ /**
+ * Lightweight listStatus API.
+ *
+ * @param keyArgs Key args
+ * @param recursive For a directory if true all the descendants of a
+ * particular directory are listed
+ * @param startKey Key from which listing needs to start. If startKey
exists
+ * its status is included in the final list.
+ * @param numEntries Number of entries to list from the start key
+ * @param allowPartialPrefixes if partial prefixes should be allowed,
+ * this is needed in context of ListKeys
+ * @return list of file status
+ */
+ List<OzoneFileStatusLight> listStatusLight(OmKeyArgs keyArgs,
+ boolean recursive, String startKey, long numEntries,
+ boolean allowPartialPrefixes) throws IOException;
+
/**
* Add acl for Ozone object. Return true if acl is added successfully else
* false.
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 7133ae03e4..567fbdad17 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.ozone.om.helpers.OmTenantArgs;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
@@ -134,6 +135,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMul
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusLightResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest;
@@ -157,6 +159,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMReque
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProtoLight;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrintCompactionLogDagRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncResponse;
@@ -2222,6 +2225,43 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
return statusList;
}
+ @Override
+ public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
+ boolean recursive, String startKey, long numEntries,
+ boolean allowPartialPrefixes) throws IOException {
+ KeyArgs keyArgs = KeyArgs.newBuilder()
+ .setVolumeName(args.getVolumeName())
+ .setBucketName(args.getBucketName())
+ .setKeyName(args.getKeyName())
+ .setSortDatanodes(false)
+ .setLatestVersionLocation(true)
+ .build();
+ ListStatusRequest.Builder listStatusRequestBuilder =
+ ListStatusRequest.newBuilder()
+ .setKeyArgs(keyArgs)
+ .setRecursive(recursive)
+ .setStartKey(startKey)
+ .setNumEntries(numEntries);
+
+ if (allowPartialPrefixes) {
+ listStatusRequestBuilder.setAllowPartialPrefix(allowPartialPrefixes);
+ }
+
+ OMRequest omRequest = createOMRequest(Type.ListStatusLight)
+ .setListStatusRequest(listStatusRequestBuilder.build())
+ .build();
+ ListStatusLightResponse listStatusLightResponse =
+ handleError(submitRequest(omRequest)).getListStatusLightResponse();
+ List<OzoneFileStatusLight> statusList =
+ new ArrayList<>(listStatusLightResponse.getStatusesCount());
+ for (OzoneFileStatusProtoLight fileStatus : listStatusLightResponse
+ .getStatusesList()) {
+
+ statusList.add(OzoneFileStatusLight.getFromProtobuf(fileStatus));
+ }
+ return statusList;
+ }
+
@Override
public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
String startKey, long numEntries) throws IOException {
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 8025fc42bb..a3fbdb2258 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -144,6 +144,7 @@ enum Type {
ListKeysLight = 126;
AbortExpiredMultiPartUploads = 127;
SetSnapshotProperty = 128;
+ ListStatusLight = 129;
}
enum SafeMode {
@@ -397,6 +398,7 @@ message OMResponse {
optional ListKeysLightResponse listKeysLightResponse =
126;
optional MultipartUploadsExpiredAbortResponse
multipartUploadsExpiredAbortResponse = 127;
optional SetSnapshotPropertyResponse SetSnapshotPropertyResponse =
128;
+ optional ListStatusLightResponse listStatusLightResponse =
129;
}
enum Status {
@@ -1129,6 +1131,14 @@ message OzoneFileStatusProto {
}
+message OzoneFileStatusProtoLight {
+ optional string volumeName = 1;
+ optional string bucketName = 2;
+ optional BasicKeyInfo basicKeyInfo = 3;
+ optional uint64 blockSize = 4;
+ optional bool isDirectory = 5;
+}
+
message GetFileStatusRequest {
required KeyArgs keyArgs = 1;
}
@@ -1137,6 +1147,10 @@ message GetFileStatusResponse {
required OzoneFileStatusProto status = 1;
}
+message GetFileStatusLightResponse {
+ required OzoneFileStatusProtoLight status = 1;
+}
+
message CreateDirectoryRequest {
required KeyArgs keyArgs = 1;
}
@@ -1182,6 +1196,10 @@ message ListStatusResponse {
repeated OzoneFileStatusProto statuses = 1;
}
+message ListStatusLightResponse {
+ repeated OzoneFileStatusProtoLight statuses = 1;
+}
+
message CreateKeyRequest {
required KeyArgs keyArgs = 1;
// Set in OM HA during preExecute step. This way all OM's use same ID in
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
index d63eea81f6..141ae104f8 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@ -37,6 +37,7 @@ import
org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
import org.apache.hadoop.ozone.security.acl.OzoneObjInfo;
import org.apache.hadoop.ozone.security.acl.RequestContext;
@@ -251,6 +252,18 @@ public class OmMetadataReader implements
IOmMetadataReader, Auditor {
}
}
}
+
+ @Override
+ public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
+ boolean recursive, String startKey, long numEntries,
+ boolean allowPartialPrefixes) throws IOException {
+ List<OzoneFileStatus> ozoneFileStatuses =
+ listStatus(args, recursive, startKey, numEntries,
allowPartialPrefixes);
+
+ return ozoneFileStatuses.stream()
+ .map(OzoneFileStatusLight::fromOzoneFileStatus)
+ .collect(Collectors.toList());
+ }
@Override
public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
index a8e3f5c5d3..5839c61cf3 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
import org.apache.hadoop.ozone.security.acl.OzoneAuthorizerFactory;
@@ -123,6 +124,19 @@ public class OmSnapshot implements IOmMetadataReader,
Closeable {
.collect(Collectors.toList());
}
+ @Override
+ public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
+ boolean recursive, String startKey, long numEntries,
+ boolean allowPartialPrefixes) throws IOException {
+
+ List<OzoneFileStatus> ozoneFileStatuses =
+ listStatus(args, recursive, startKey, numEntries,
allowPartialPrefixes);
+
+ return ozoneFileStatuses.stream()
+ .map(OzoneFileStatusLight::fromOzoneFileStatus)
+ .collect(Collectors.toList());
+ }
+
@Override
public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException {
return denormalizeOzoneFileStatus(
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 26ef8cdd33..e67f4e505c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -162,6 +162,7 @@ import
org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
@@ -3642,6 +3643,18 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
}
+ @Override
+ public List<OzoneFileStatusLight> listStatusLight(OmKeyArgs args,
+ boolean recursive, String startKey, long numEntries,
+ boolean allowPartialPrefixes) throws IOException {
+ List<OzoneFileStatus> ozoneFileStatuses =
+ listStatus(args, recursive, startKey, numEntries,
allowPartialPrefixes);
+
+ return ozoneFileStatuses.stream()
+ .map(OzoneFileStatusLight::fromOzoneFileStatus)
+ .collect(Collectors.toList());
+ }
+
/**
* {@inheritDoc}
*/
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 30333f735e..0e80632bd7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -55,6 +55,7 @@ import
org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
import org.apache.hadoop.ozone.om.helpers.OmPartInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx;
@@ -146,6 +147,7 @@ import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsResponse;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusRequest;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusResponse;
+import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusLightResponse;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileRequest;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LookupFileResponse;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadInfo;
@@ -271,6 +273,12 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
listStatus(request.getListStatusRequest(), request.getVersion());
responseBuilder.setListStatusResponse(listStatusResponse);
break;
+ case ListStatusLight:
+ ListStatusLightResponse listStatusLightResponse =
+ listStatusLight(request.getListStatusRequest(),
+ request.getVersion());
+ responseBuilder.setListStatusLightResponse(listStatusLightResponse);
+ break;
case GetAcl:
GetAclResponse getAclResponse =
getAcl(request.getGetAclRequest());
@@ -1178,6 +1186,32 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
return listStatusResponseBuilder.build();
}
+ private ListStatusLightResponse listStatusLight(
+ ListStatusRequest request, int clientVersion) throws IOException {
+ KeyArgs keyArgs = request.getKeyArgs();
+ OmKeyArgs omKeyArgs = new OmKeyArgs.Builder()
+ .setVolumeName(keyArgs.getVolumeName())
+ .setBucketName(keyArgs.getBucketName())
+ .setKeyName(keyArgs.getKeyName())
+ .setSortDatanodesInPipeline(false)
+ .setLatestVersionLocation(true)
+ .setHeadOp(keyArgs.getHeadOp())
+ .build();
+ boolean allowPartialPrefixes =
+ request.hasAllowPartialPrefix() && request.getAllowPartialPrefix();
+ List<OzoneFileStatusLight> statuses =
+ impl.listStatusLight(omKeyArgs, request.getRecursive(),
+ request.getStartKey(), request.getNumEntries(),
+ allowPartialPrefixes);
+ ListStatusLightResponse.Builder
+ listStatusLightResponseBuilder =
+ ListStatusLightResponse.newBuilder();
+ for (OzoneFileStatusLight status : statuses) {
+ listStatusLightResponseBuilder.addStatuses(status.getProtobuf());
+ }
+ return listStatusLightResponseBuilder.build();
+ }
+
@RequestFeatureValidator(
conditions = ValidationCondition.OLDER_CLIENT_REQUESTS,
processingPhase = RequestProcessingPhase.POST_PROCESS,
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index 4eeb2ecaa1..5505688b26 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadCompleteInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
+import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.helpers.S3VolumeContext;
@@ -519,6 +520,13 @@ public class ClientProtocolStub implements ClientProtocol {
return null;
}
+ @Override
+ public List<OzoneFileStatusLight> listStatusLight(String volumeName,
+ String bucketName, String keyName, boolean recursive, String startKey,
+ long numEntries, boolean allowPartialPrefixes) throws IOException {
+ return null;
+ }
+
@Override
public boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException {
return false;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
