This is an automated email from the ASF dual-hosted git repository.
ritesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ad176eb0c1 HDDS-9079. Implement a lightweight listKeys API (#5115)
ad176eb0c1 is described below
commit ad176eb0c10ced990d2063cb1b6d4004c77dbe55
Author: tanvipenumudy <[email protected]>
AuthorDate: Wed Sep 27 01:29:02 2023 +0530
HDDS-9079. Implement a lightweight listKeys API (#5115)
---
.../apache/hadoop/ozone/OzoneManagerVersion.java | 3 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 42 ++--
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 +
.../apache/hadoop/ozone/om/IOmMetadataReader.java | 30 ++-
.../hadoop/ozone/om/helpers/BasicOmKeyInfo.java | 222 +++++++++++++++++++++
.../ozone/om/helpers/ListKeysLightResult.java | 43 ++++
.../hadoop/ozone/om/helpers/ListKeysResult.java | 42 ++++
...OzoneManagerProtocolClientSideTranslatorPB.java | 47 ++++-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 5 +-
.../hadoop/ozone/om/TestOzoneManagerPrepare.java | 2 +-
.../src/main/proto/OmClientProtocol.proto | 18 ++
.../apache/hadoop/ozone/om/OMMetadataManager.java | 6 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 5 +-
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 11 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 12 +-
.../apache/hadoop/ozone/om/OmMetadataReader.java | 21 +-
.../org/apache/hadoop/ozone/om/OmSnapshot.java | 30 ++-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 25 ++-
.../protocolPB/OzoneManagerRequestHandler.java | 31 ++-
.../hadoop/ozone/om/TestOmMetadataManager.java | 20 +-
.../hadoop/ozone/om/TestSstFilteringService.java | 2 +-
.../hadoop/ozone/freon/OmMetadataGenerator.java | 5 +-
22 files changed, 568 insertions(+), 55 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
index a91885e25d..00f2e55f97 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java
@@ -37,6 +37,9 @@ public enum OzoneManagerVersion implements ComponentVersion {
OPTIMIZED_GET_KEY_INFO(3, "OzoneManager version that supports optimized"
+ " key lookups using cached container locations."),
+ LIGHTWEIGHT_LIST_KEYS(4, "OzoneManager version that supports lightweight"
+ + " listKeys API."),
+
FUTURE_VERSION(-1, "Used internally in the client when the server side is "
+ " newer and an unknown server version has arrived to the client.");
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 183a62c1ea..d3a3106934 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -107,6 +107,7 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
import org.apache.hadoop.ozone.om.helpers.DeleteTenantState;
@@ -1558,18 +1559,35 @@ public class RpcClient implements ClientProtocol {
String keyPrefix, String prevKey,
int maxListResult)
throws IOException {
- List<OmKeyInfo> keys = ozoneManagerClient.listKeys(
- volumeName, bucketName, prevKey, keyPrefix, maxListResult);
- return keys.stream().map(key -> new OzoneKey(
- key.getVolumeName(),
- key.getBucketName(),
- key.getKeyName(),
- key.getDataSize(),
- key.getCreationTime(),
- key.getModificationTime(),
- key.getReplicationConfig(),
- key.isFile()))
- .collect(Collectors.toList());
+
+ if (omVersion.compareTo(OzoneManagerVersion.LIGHTWEIGHT_LIST_KEYS) >= 0) {
+ List<BasicOmKeyInfo> keys = ozoneManagerClient.listKeysLight(
+ volumeName, bucketName, prevKey, keyPrefix, maxListResult).getKeys();
+
+ return keys.stream().map(key -> new OzoneKey(
+ key.getVolumeName(),
+ key.getBucketName(),
+ key.getKeyName(),
+ key.getDataSize(),
+ key.getCreationTime(),
+ key.getModificationTime(),
+ key.getReplicationConfig(),
+ key.isFile()))
+ .collect(Collectors.toList());
+ } else {
+ List<OmKeyInfo> keys = ozoneManagerClient.listKeys(
+ volumeName, bucketName, prevKey, keyPrefix, maxListResult).getKeys();
+ return keys.stream().map(key -> new OzoneKey(
+ key.getVolumeName(),
+ key.getBucketName(),
+ key.getKeyName(),
+ key.getDataSize(),
+ key.getCreationTime(),
+ key.getModificationTime(),
+ key.getReplicationConfig(),
+ key.isFile()))
+ .collect(Collectors.toList());
+ }
}
@Override
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 5a293d3256..0e0a89a7a2 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -241,6 +241,7 @@ public final class OmUtils {
case ListBuckets:
case LookupKey:
case ListKeys:
+ case ListKeysLight:
case ListTrash:
case ServiceList:
case ListMultiPartUploadParts:
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
index 9bdc4772aa..c0a920aca9 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/IOmMetadataReader.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
@@ -109,8 +111,32 @@ public interface IOmMetadataReader {
* the size of the result will not exceed this limit.
* @return a list of keys.
*/
- List<OmKeyInfo> listKeys(String volumeName, String bucketName,
- String startKey, String keyPrefix, int maxKeys)
+ ListKeysResult listKeys(String volumeName, String bucketName,
+ String startKey, String keyPrefix, int maxKeys)
+ throws IOException;
+
+ /**
+ * Lightweight listKeys implementation.
+ *
+ * @param volumeName
+ * the name of the volume.
+ * @param bucketName
+ * the name of the bucket.
+ * @param startKey
+ * the start key name, only the keys whose name is
+ * after this value will be included in the result.
+ * @param keyPrefix
+ * key name prefix, only the keys whose name has
+ * this prefix will be included in the result.
+ * @param maxKeys
+ * the maximum number of keys to return. It ensures
+ * the size of the result will not exceed this limit.
+ * @return a list of keys.
+ * @throws IOException
+ */
+ ListKeysLightResult listKeysLight(String volumeName, String bucketName,
+ String startKey, String keyPrefix,
+ int maxKeys)
throws IOException;
/**
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java
new file mode 100644
index 0000000000..2d7d859832
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.helpers;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BasicKeyInfo;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest;
+
+/**
+ * Lightweight OmKeyInfo class.
+ */
+public class BasicOmKeyInfo {
+
+ private String volumeName;
+ private String bucketName;
+ private String keyName;
+ private long dataSize;
+ private long creationTime;
+ private long modificationTime;
+ private ReplicationConfig replicationConfig;
+ private boolean isFile;
+
+ @SuppressWarnings("parameternumber")
+ public BasicOmKeyInfo(String volumeName, String bucketName, String keyName,
+ long dataSize, long creationTime, long
modificationTime,
+ ReplicationConfig replicationConfig, boolean isFile) {
+ this.volumeName = volumeName;
+ this.bucketName = bucketName;
+ this.keyName = keyName;
+ this.dataSize = dataSize;
+ this.creationTime = creationTime;
+ this.modificationTime = modificationTime;
+ this.replicationConfig = replicationConfig;
+ this.isFile = isFile;
+ }
+
+ public String getVolumeName() {
+ return volumeName;
+ }
+
+ public String getBucketName() {
+ return bucketName;
+ }
+
+ public String getKeyName() {
+ return keyName;
+ }
+
+ public long getDataSize() {
+ return dataSize;
+ }
+
+ public long getCreationTime() {
+ return creationTime;
+ }
+
+ public long getModificationTime() {
+ return modificationTime;
+ }
+
+ public ReplicationConfig getReplicationConfig() {
+ return replicationConfig;
+ }
+
+ public boolean isFile() {
+ return isFile;
+ }
+
+ /**
+ * Builder of BasicOmKeyInfo.
+ */
+ public static class Builder {
+ private String volumeName;
+ private String bucketName;
+ private String keyName;
+ private long dataSize;
+ private long creationTime;
+ private long modificationTime;
+ private ReplicationConfig replicationConfig;
+ private boolean isFile;
+
+ public Builder setVolumeName(String volumeName) {
+ this.volumeName = volumeName;
+ return this;
+ }
+
+ public Builder setBucketName(String bucketName) {
+ this.bucketName = bucketName;
+ return this;
+ }
+
+ public Builder setKeyName(String keyName) {
+ this.keyName = keyName;
+ return this;
+ }
+
+ public Builder setDataSize(long dataSize) {
+ this.dataSize = dataSize;
+ return this;
+ }
+
+ public Builder setCreationTime(long creationTime) {
+ this.creationTime = creationTime;
+ return this;
+ }
+
+ public Builder setModificationTime(long modificationTime) {
+ this.modificationTime = modificationTime;
+ return this;
+ }
+
+ public Builder setReplicationConfig(ReplicationConfig replicationConfig) {
+ this.replicationConfig = replicationConfig;
+ return this;
+ }
+
+ public Builder setIsFile(boolean isFile) {
+ this.isFile = isFile;
+ return this;
+ }
+
+ public BasicOmKeyInfo build() {
+ return new BasicOmKeyInfo(volumeName, bucketName, keyName, dataSize,
+ creationTime, modificationTime, replicationConfig, isFile);
+ }
+ }
+
+ public BasicKeyInfo getProtobuf() {
+ BasicKeyInfo.Builder builder = BasicKeyInfo.newBuilder()
+ .setKeyName(keyName)
+ .setDataSize(dataSize)
+ .setCreationTime(creationTime)
+ .setModificationTime(modificationTime)
+ .setType(replicationConfig.getReplicationType());
+ if (replicationConfig instanceof ECReplicationConfig) {
+ builder.setEcReplicationConfig(
+ ((ECReplicationConfig) replicationConfig).toProto());
+ } else {
+ builder.setFactor(ReplicationConfig.getLegacyFactor(replicationConfig));
+ }
+
+ return builder.build();
+ }
+
+ public static BasicOmKeyInfo getFromProtobuf(BasicKeyInfo basicKeyInfo,
+ ListKeysRequest request)
+ throws IOException {
+ if (basicKeyInfo == null || request == null) {
+ return null;
+ }
+
+ String keyName = basicKeyInfo.getKeyName();
+
+ Builder builder = new Builder()
+ .setVolumeName(request.getVolumeName())
+ .setBucketName(request.getBucketName())
+ .setKeyName(keyName)
+ .setDataSize(basicKeyInfo.getDataSize())
+ .setCreationTime(basicKeyInfo.getCreationTime())
+ .setModificationTime(basicKeyInfo.getModificationTime())
+ .setReplicationConfig(ReplicationConfig.fromProto(
+ basicKeyInfo.getType(),
+ basicKeyInfo.getFactor(),
+ basicKeyInfo.getEcReplicationConfig()))
+ .setIsFile(!keyName.endsWith("/"));
+
+ return builder.build();
+ }
+
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ BasicOmKeyInfo basicOmKeyInfo = (BasicOmKeyInfo) o;
+ return volumeName.equals(basicOmKeyInfo.volumeName) &&
+ bucketName.equals(basicOmKeyInfo.bucketName) &&
+ keyName.equals(basicOmKeyInfo.keyName) &&
+ dataSize == basicOmKeyInfo.dataSize &&
+ creationTime == basicOmKeyInfo.creationTime &&
+ modificationTime == basicOmKeyInfo.modificationTime &&
+ replicationConfig.equals(basicOmKeyInfo.replicationConfig) &&
+ isFile == basicOmKeyInfo.isFile;
+ }
+
+ public int hashCode() {
+ return Objects.hash(volumeName, bucketName, keyName);
+ }
+
+ public static BasicOmKeyInfo fromOmKeyInfo(OmKeyInfo omKeyInfo) {
+ return new BasicOmKeyInfo(
+ omKeyInfo.getVolumeName(),
+ omKeyInfo.getBucketName(),
+ omKeyInfo.getKeyName(),
+ omKeyInfo.getDataSize(),
+ omKeyInfo.getCreationTime(),
+ omKeyInfo.getModificationTime(),
+ omKeyInfo.getReplicationConfig(),
+ omKeyInfo.isFile());
+ }
+}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysLightResult.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysLightResult.java
new file mode 100644
index 0000000000..47a0669cc3
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysLightResult.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.helpers;
+
+import java.util.List;
+
+/**
+ * Encapsulates the result of listKeys. It contains a list of
+ * {@link BasicOmKeyInfo} and a boolean flag indicating if the listing is
+ * truncated.
+ */
+public class ListKeysLightResult {
+ private List<BasicOmKeyInfo> keys;
+ private boolean isTruncated;
+
+ public ListKeysLightResult(List<BasicOmKeyInfo> keys, boolean isTruncated) {
+ this.keys = keys;
+ this.isTruncated = isTruncated;
+ }
+
+ public List<BasicOmKeyInfo> getKeys() {
+ return keys;
+ }
+
+ public boolean isTruncated() {
+ return isTruncated;
+ }
+}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysResult.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysResult.java
new file mode 100644
index 0000000000..679e1a25e9
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ListKeysResult.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.helpers;
+
+import java.util.List;
+
+/**
+ * Encapsulates the result of listKeys. It contains a list of {@link OmKeyInfo}
+ * and a boolean flag indicating if the listing is truncated.
+ */
+public class ListKeysResult {
+ private List<OmKeyInfo> keys;
+ private boolean isTruncated;
+
+ public ListKeysResult(List<OmKeyInfo> keys, boolean isTruncated) {
+ this.keys = keys;
+ this.isTruncated = isTruncated;
+ }
+
+ public List<OmKeyInfo> getKeys() {
+ return keys;
+ }
+
+ public boolean isTruncated() {
+ return isTruncated;
+ }
+}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index a4820817c0..7133ae03e4 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -40,6 +40,9 @@ import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.DeleteTenantState;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
@@ -126,6 +129,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBuc
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBucketsResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysLightResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListMultipartUploadsResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListStatusRequest;
@@ -971,7 +975,7 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
* List keys in a bucket.
*/
@Override
- public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
+ public ListKeysResult listKeys(String volumeName, String bucketName,
String startKey, String prefix, int maxKeys) throws IOException {
List<OmKeyInfo> keys = new ArrayList<>();
ListKeysRequest.Builder reqBuilder = ListKeysRequest.newBuilder();
@@ -1001,10 +1005,49 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
list.add(fromProtobuf);
}
keys.addAll(list);
- return keys;
+ return new ListKeysResult(keys, resp.getIsTruncated());
}
+ /**
+ * Lightweight listKeys implementation.
+ */
+ @Override
+ public ListKeysLightResult listKeysLight(String volumeName,
+ String bucketName, String startKey,
+ String prefix,
+ int maxKeys) throws IOException {
+ List<BasicOmKeyInfo> keys = new ArrayList<>();
+ ListKeysRequest.Builder reqBuilder = ListKeysRequest.newBuilder();
+ reqBuilder.setVolumeName(volumeName);
+ reqBuilder.setBucketName(bucketName);
+ reqBuilder.setCount(maxKeys);
+
+ if (StringUtils.isNotEmpty(startKey)) {
+ reqBuilder.setStartKey(startKey);
+ }
+
+ if (prefix != null) {
+ reqBuilder.setPrefix(prefix);
+ }
+
+ ListKeysRequest req = reqBuilder.build();
+
+ OMRequest omRequest = createOMRequest(Type.ListKeysLight)
+ .setListKeysRequest(req)
+ .build();
+
+ ListKeysLightResponse resp =
+ handleError(submitRequest(omRequest)).getListKeysLightResponse();
+ for (OzoneManagerProtocolProtos.BasicKeyInfo
+ basicKeyInfo : resp.getBasicKeyInfoList()) {
+ BasicOmKeyInfo fromProtobuf =
+ BasicOmKeyInfo.getFromProtobuf(basicKeyInfo, req);
+ keys.add(fromProtobuf);
+ }
+ return new ListKeysLightResult(keys, resp.getIsTruncated());
+ }
+
@Override
@Nonnull
public S3SecretValue getS3Secret(String kerberosID) throws IOException {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
index f9d83c8390..e2ce4e9ba5 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
@@ -953,8 +953,9 @@ public class TestKeyManagerImpl {
assertEquals(key.getKeyLocationVersions().size(), 2);
// Test ListKeys (latestLocationVersion is always true for ListKeys)
- List<OmKeyInfo> keyInfos = keyManager.listKeys(keyArgs.getVolumeName(),
- keyArgs.getBucketName(), "", keyArgs.getKeyName(), 100);
+ List<OmKeyInfo> keyInfos =
+ keyManager.listKeys(keyArgs.getVolumeName(), keyArgs.getBucketName(),
+ "", keyArgs.getKeyName(), 100).getKeys();
assertEquals(keyInfos.size(), 1);
assertEquals(keyInfos.get(0).getKeyLocationVersions().size(), 1);
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java
index 15f2bd9031..20b343dbf9 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerPrepare.java
@@ -411,7 +411,7 @@ public class TestOzoneManagerPrepare extends
TestOzoneManagerHA {
// Wait for a potentially slow follower to apply all key writes.
LambdaTestUtils.await(WAIT_TIMEOUT_MILLIS, 1000, () -> {
List<OmKeyInfo> keys = om.getMetadataManager().listKeys(volumeName,
- BUCKET, null, KEY_PREFIX, 100);
+ BUCKET, null, KEY_PREFIX, 100).getKeys();
boolean allKeysFound = (expectedKeys.size() == keys.size());
if (!allKeysFound) {
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 85a6af4d35..01d384d5c2 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -141,6 +141,7 @@ enum Type {
CancelSnapshotDiff = 123;
SetSafeMode = 124;
PrintCompactionLogDag = 125;
+ ListKeysLight = 126;
}
enum SafeMode {
@@ -388,6 +389,7 @@ message OMResponse {
optional CancelSnapshotDiffResponse cancelSnapshotDiffResponse =
123;
optional SetSafeModeResponse SetSafeModeResponse =
124;
optional PrintCompactionLogDagResponse PrintCompactionLogDagResponse =
125;
+ optional ListKeysLightResponse listKeysLightResponse =
126;
}
enum Status {
@@ -1085,6 +1087,16 @@ message KeyInfo {
optional bool isFile = 19;
}
+message BasicKeyInfo {
+ optional string keyName = 1;
+ optional uint64 dataSize = 2;
+ optional uint64 creationTime = 3;
+ optional uint64 modificationTime = 4;
+ optional hadoop.hdds.ReplicationType type = 5;
+ optional hadoop.hdds.ReplicationFactor factor = 6;
+ optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 7;
+}
+
message DirectoryInfo {
required string name = 1;
required uint64 creationTime = 2;
@@ -1358,6 +1370,12 @@ message ListKeysRequest {
message ListKeysResponse {
repeated KeyInfo keyInfo = 2;
+ optional bool isTruncated = 3;
+}
+
+message ListKeysLightResponse {
+ repeated BasicKeyInfo basicKeyInfo = 1;
+ optional bool isTruncated = 2;
}
message CommitKeyRequest {
diff --git
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 0832344dce..66baf41184 100644
---
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.common.BlockGroup;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo;
import org.apache.hadoop.ozone.om.helpers.OmDBUserPrincipalInfo;
@@ -202,8 +203,9 @@ public interface OMMetadataManager extends DBStoreHAManager
{
* @return a list of keys.
* @throws IOException
*/
- List<OmKeyInfo> listKeys(String volumeName,
- String bucketName, String startKey, String keyPrefix, int maxKeys)
+ ListKeysResult listKeys(String volumeName,
+ String bucketName, String startKey, String keyPrefix,
+ int maxKeys)
throws IOException;
/**
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
index 0bcc7f8231..6ba98cbde0 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
@@ -98,8 +99,8 @@ public interface KeyManager extends OzoneManagerFS, IOzoneAcl
{
* @return a list of keys.
* @throws IOException
*/
- List<OmKeyInfo> listKeys(String volumeName,
- String bucketName, String startKey, String keyPrefix, int maxKeys)
+ ListKeysResult listKeys(String volumeName, String bucketName, String
startKey,
+ String keyPrefix, int maxKeys)
throws IOException;
/**
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 918cb3ec56..33ea1f779b 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
@@ -556,7 +557,7 @@ public class KeyManagerImpl implements KeyManager {
}
@Override
- public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
+ public ListKeysResult listKeys(String volumeName, String bucketName,
String startKey, String keyPrefix,
int maxKeys) throws IOException {
Preconditions.checkNotNull(volumeName);
@@ -572,15 +573,17 @@ public class KeyManagerImpl implements KeyManager {
keyPrefix = OmUtils.normalizeKey(keyPrefix, true);
}
- List<OmKeyInfo> keyList = metadataManager.listKeys(volumeName, bucketName,
- startKey, keyPrefix, maxKeys);
+ ListKeysResult listKeysResult =
+ metadataManager.listKeys(volumeName, bucketName, startKey, keyPrefix,
+ maxKeys);
+ List<OmKeyInfo> keyList = listKeysResult.getKeys();
// For listKeys, we return the latest Key Location by default
for (OmKeyInfo omKeyInfo : keyList) {
slimLocationVersion(omKeyInfo);
}
- return keyList;
+ return listKeysResult;
}
@Override
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 512ff167b6..6f0fafcff2 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.om.codec.TokenIdentifierCodec;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo;
import org.apache.hadoop.ozone.om.helpers.OmDBUserPrincipalInfo;
@@ -1168,12 +1169,13 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
}
@Override
- public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
- String startKey, String keyPrefix, int maxKeys) throws IOException {
+ public ListKeysResult listKeys(String volumeName, String bucketName,
+ String startKey, String keyPrefix, int
maxKeys)
+ throws IOException {
List<OmKeyInfo> result = new ArrayList<>();
if (maxKeys <= 0) {
- return result;
+ return new ListKeysResult(result, false);
}
if (Strings.isNullOrEmpty(volumeName)) {
@@ -1268,6 +1270,8 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
}
}
+ boolean isTruncated = cacheKeyMap.size() > maxKeys;
+
// Finally DB entries and cache entries are merged, then return the count
// of maxKeys from the sorted map.
currentCount = 0;
@@ -1288,7 +1292,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager,
// Clear map and set.
cacheKeyMap.clear();
- return result;
+ return new ListKeysResult(result, isTruncated);
}
// TODO: HDDS-2419 - Complete stub below for core logic
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
index 72506d9821..467b23df52 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java
@@ -30,6 +30,9 @@ import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.audit.Auditor;
import org.apache.hadoop.ozone.audit.OMAction;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -43,6 +46,7 @@ import org.slf4j.Logger;
import java.net.InetAddress;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import static org.apache.hadoop.hdds.server.ServerUtils.getRemoteUserName;
import static org.apache.hadoop.hdds.utils.HddsServerUtil.getRemoteUser;
@@ -305,7 +309,7 @@ public class OmMetadataReader implements IOmMetadataReader,
Auditor {
}
@Override
- public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
+ public ListKeysResult listKeys(String volumeName, String bucketName,
String startKey, String keyPrefix, int maxKeys) throws IOException {
long startNanos = Time.monotonicNowNanos();
ResolvedBucket bucket = ozoneManager.resolveBucketLink(
@@ -340,6 +344,21 @@ public class OmMetadataReader implements
IOmMetadataReader, Auditor {
}
}
+ @Override
+ public ListKeysLightResult listKeysLight(String volumeName,
+ String bucketName,
+ String startKey, String keyPrefix,
+ int maxKeys) throws IOException {
+ ListKeysResult listKeysResult =
+ listKeys(volumeName, bucketName, startKey, keyPrefix, maxKeys);
+ List<OmKeyInfo> keys = listKeysResult.getKeys();
+ List<BasicOmKeyInfo> basicKeysList =
+ keys.stream().map(BasicOmKeyInfo::fromOmKeyInfo)
+ .collect(Collectors.toList());
+
+ return new ListKeysLightResult(basicKeysList,
listKeysResult.isTruncated());
+ }
+
/**
* Returns list of ACLs for given Ozone object.
*
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
index 8fa9bc25f1..a8e3f5c5d3 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshot.java
@@ -23,6 +23,9 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.audit.AuditLogger;
import org.apache.hadoop.ozone.audit.AuditLoggerType;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -133,12 +136,29 @@ public class OmSnapshot implements IOmMetadataReader,
Closeable {
}
@Override
- public List<OmKeyInfo> listKeys(String vname, String bname,
- String startKey, String keyPrefix, int maxKeys) throws IOException {
- List<OmKeyInfo> l = omMetadataReader.listKeys(vname, bname,
+ public ListKeysResult listKeys(String vname, String bname,
+ String startKey, String keyPrefix, int
maxKeys)
+ throws IOException {
+ ListKeysResult listKeysResult = omMetadataReader.listKeys(vname, bname,
normalizeKeyName(startKey), normalizeKeyName(keyPrefix), maxKeys);
- return l.stream().map(this::denormalizeOmKeyInfo)
- .collect(Collectors.toList());
+ return new ListKeysResult(
+ listKeysResult.getKeys().stream().map(this::denormalizeOmKeyInfo)
+ .collect(Collectors.toList()), listKeysResult.isTruncated());
+ }
+
+ @Override
+ public ListKeysLightResult listKeysLight(String volName,
+ String buckName,
+ String startKey, String keyPrefix,
+ int maxKeys) throws IOException {
+ ListKeysResult listKeysResult =
+ listKeys(volumeName, bucketName, startKey, keyPrefix, maxKeys);
+ List<OmKeyInfo> keys = listKeysResult.getKeys();
+ List<BasicOmKeyInfo> basicKeysList =
+ keys.stream().map(BasicOmKeyInfo::fromOmKeyInfo)
+ .collect(Collectors.toList());
+
+ return new ListKeysLightResult(basicKeysList,
listKeysResult.isTruncated());
}
@Override
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 40b8e94c74..e627d97bc6 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -154,6 +154,9 @@ import org.apache.hadoop.ozone.om.helpers.OmDBAccessIdInfo;
import org.apache.hadoop.ozone.om.helpers.OmDBUserPrincipalInfo;
import org.apache.hadoop.ozone.om.helpers.OmDBTenantState;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
+import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
@@ -2835,15 +2838,31 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
* {@inheritDoc}
*/
@Override
- public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
- String startKey, String keyPrefix, int maxKeys) throws IOException {
+ public ListKeysResult listKeys(String volumeName, String bucketName,
+ String startKey, String keyPrefix, int
maxKeys)
+ throws IOException {
try (ReferenceCounted<IOmMetadataReader, SnapshotCache> rcReader =
- getReader(volumeName, bucketName, keyPrefix)) {
+ getReader(volumeName, bucketName, keyPrefix)) {
return rcReader.get().listKeys(
volumeName, bucketName, startKey, keyPrefix, maxKeys);
}
}
+ @Override
+ public ListKeysLightResult listKeysLight(String volumeName,
+ String bucketName,
+ String startKey, String keyPrefix,
+ int maxKeys) throws IOException {
+ ListKeysResult listKeysResult =
+ listKeys(volumeName, bucketName, startKey, keyPrefix, maxKeys);
+ List<OmKeyInfo> keys = listKeysResult.getKeys();
+ List<BasicOmKeyInfo> basicKeysList =
+ keys.stream().map(BasicOmKeyInfo::fromOmKeyInfo)
+ .collect(Collectors.toList());
+
+ return new ListKeysLightResult(basicKeysList,
listKeysResult.isTruncated());
+ }
+
@Override
public List<RepeatedOmKeyInfo> listTrash(String volumeName,
String bucketName, String startKeyName, String keyPrefix, int maxKeys)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 08ba705495..080f48cfcc 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -42,6 +42,9 @@ import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.OzoneManagerPrepareState;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
+import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -98,6 +101,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBuc
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListBucketsResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListKeysLightResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTenantResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ListTrashRequest;
@@ -216,6 +220,11 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
request.getListKeysRequest(), request.getVersion());
responseBuilder.setListKeysResponse(listKeysResponse);
break;
+ case ListKeysLight:
+ ListKeysLightResponse listKeysLightResponse = listKeysLight(
+ request.getListKeysRequest());
+ responseBuilder.setListKeysLightResponse(listKeysLightResponse);
+ break;
case ListTrash:
ListTrashResponse listTrashResponse = listTrash(
request.getListTrashRequest(), request.getVersion());
@@ -671,16 +680,34 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
ListKeysResponse.Builder resp =
ListKeysResponse.newBuilder();
- List<OmKeyInfo> keys = impl.listKeys(
+ ListKeysResult listKeysResult = impl.listKeys(
request.getVolumeName(),
request.getBucketName(),
request.getStartKey(),
request.getPrefix(),
request.getCount());
- for (OmKeyInfo key : keys) {
+ for (OmKeyInfo key : listKeysResult.getKeys()) {
resp.addKeyInfo(key.getProtobuf(true, clientVersion));
}
+ resp.setIsTruncated(listKeysResult.isTruncated());
+ return resp.build();
+ }
+ private ListKeysLightResponse listKeysLight(ListKeysRequest request)
+ throws IOException {
+ ListKeysLightResponse.Builder resp =
+ ListKeysLightResponse.newBuilder();
+
+ ListKeysLightResult listKeysLightResult = impl.listKeysLight(
+ request.getVolumeName(),
+ request.getBucketName(),
+ request.getStartKey(),
+ request.getPrefix(),
+ request.getCount());
+ for (BasicOmKeyInfo key : listKeysLightResult.getKeys()) {
+ resp.addBasicKeyInfo(key.getProtobuf());
+ }
+ resp.setIsTruncated(listKeysLightResult.isTruncated());
return resp.build();
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
index c2a334e20b..34bd721602 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -390,7 +390,7 @@ public class TestOmMetadataManager {
// List all keys which have prefix "key-a"
List<OmKeyInfo> omKeyInfoList =
omMetadataManager.listKeys(volumeNameA, ozoneBucket,
- null, prefixKeyA, 100);
+ null, prefixKeyA, 100).getKeys();
assertEquals(omKeyInfoList.size(), 50);
@@ -403,7 +403,7 @@ public class TestOmMetadataManager {
String startKey = prefixKeyA + 10;
omKeyInfoList =
omMetadataManager.listKeys(volumeNameA, ozoneBucket,
- startKey, prefixKeyA, 100);
+ startKey, prefixKeyA, 100).getKeys();
assertEquals(keysASet.tailSet(
startKey).size() - 1, omKeyInfoList.size());
@@ -411,7 +411,7 @@ public class TestOmMetadataManager {
startKey = prefixKeyA + 38;
omKeyInfoList =
omMetadataManager.listKeys(volumeNameA, ozoneBucket,
- startKey, prefixKeyA, 100);
+ startKey, prefixKeyA, 100).getKeys();
assertEquals(keysASet.tailSet(
startKey).size() - 1, omKeyInfoList.size());
@@ -426,7 +426,7 @@ public class TestOmMetadataManager {
omKeyInfoList = omMetadataManager.listKeys(volumeNameB, hadoopBucket,
- null, prefixKeyB, 100);
+ null, prefixKeyB, 100).getKeys();
assertEquals(omKeyInfoList.size(), 50);
@@ -442,7 +442,7 @@ public class TestOmMetadataManager {
for (int i = 0; i < 5; i++) {
omKeyInfoList = omMetadataManager.listKeys(volumeNameB, hadoopBucket,
- startKey, prefixKeyB, 10);
+ startKey, prefixKeyB, 10).getKeys();
assertEquals(10, omKeyInfoList.size());
@@ -460,13 +460,13 @@ public class TestOmMetadataManager {
// As now we have iterated all 50 buckets, calling next time should
// return empty list.
omKeyInfoList = omMetadataManager.listKeys(volumeNameB, hadoopBucket,
- startKey, prefixKeyB, 10);
+ startKey, prefixKeyB, 10).getKeys();
assertEquals(omKeyInfoList.size(), 0);
// List all keys with empty prefix
omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket,
- null, null, 100);
+ null, null, 100).getKeys();
assertEquals(50, omKeyInfoList.size());
for (OmKeyInfo omKeyInfo : omKeyInfoList) {
assertTrue(omKeyInfo.getKeyName().startsWith(
@@ -509,7 +509,7 @@ public class TestOmMetadataManager {
// Now list keys which match with prefixKeyA.
List<OmKeyInfo> omKeyInfoList =
omMetadataManager.listKeys(volumeNameA, ozoneBucket,
- null, prefixKeyA, 100);
+ null, prefixKeyA, 100).getKeys();
// As in total 100, 50 are marked for delete. It should list only 50 keys.
assertEquals(50, omKeyInfoList.size());
@@ -530,7 +530,7 @@ public class TestOmMetadataManager {
for (int i = 0; i < 5; i++) {
omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket,
- startKey, prefixKeyA, 10);
+ startKey, prefixKeyA, 10).getKeys();
System.out.println(i);
assertEquals(10, omKeyInfoList.size());
@@ -549,7 +549,7 @@ public class TestOmMetadataManager {
// As now we have iterated all 50 buckets, calling next time should
// return empty list.
omKeyInfoList = omMetadataManager.listKeys(volumeNameA, ozoneBucket,
- startKey, prefixKeyA, 10);
+ startKey, prefixKeyA, 10).getKeys();
assertEquals(omKeyInfoList.size(), 0);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java
index a8c126f43b..7a17c70ef7 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java
@@ -482,7 +482,7 @@ public class TestSstFilteringService {
String startKey = null;
while (true) {
List<OmKeyInfo> omKeyInfoList = omMetadataReader.listKeys(volume, bucket,
- startKey, null, 1000);
+ startKey, null, 1000).getKeys();
if (omKeyInfoList.isEmpty()) {
break;
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmMetadataGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmMetadataGenerator.java
index 6ce460563f..106bf9d2ec 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmMetadataGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmMetadataGenerator.java
@@ -396,8 +396,9 @@ public class OmMetadataGenerator extends BaseFreonGenerator
threadSeqId = getThreadSequenceId();
startKeyName = getPath(threadSeqId * batchSize);
getMetrics().timer(operation.name()).time(() -> {
- List<OmKeyInfo> keyInfoList = ozoneManagerClient.listKeys(
- volumeName, bucketName, startKeyName, "", batchSize);
+ List<OmKeyInfo> keyInfoList =
+ ozoneManagerClient.listKeys(volumeName, bucketName, startKeyName,
+ "", batchSize).getKeys();
if (keyInfoList.size() + 1 < batchSize) {
throw new NoSuchFileException(
"There are not enough files for testing you should use "
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
