This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch HDDS-6517-Snapshot
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-6517-Snapshot by this
push:
new 9f84c77613 HDDS-7507. [Snapshot] Implement List Snapshot API
Pagination (#4065)
9f84c77613 is described below
commit 9f84c77613c66f5b0c24bb6b41eb13c29ecf8fcc
Author: Chung En Lee <[email protected]>
AuthorDate: Tue Mar 7 03:02:07 2023 +0800
HDDS-7507. [Snapshot] Implement List Snapshot API Pagination (#4065)
---
.../apache/hadoop/ozone/client/ObjectStore.java | 79 ++++++++++++++++++--
.../ozone/client/protocol/ClientProtocol.java | 12 ++--
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 ++--
.../ozone/om/protocol/OzoneManagerProtocol.java | 12 ++--
...OzoneManagerProtocolClientSideTranslatorPB.java | 16 ++++-
.../client/rpc/TestOzoneRpcClientAbstract.java | 83 ++++++++++++++++++++++
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 3 +-
.../src/main/proto/OmClientProtocol.proto | 3 +
.../apache/hadoop/ozone/om/OMMetadataManager.java | 12 ++--
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 60 ++++++++++++----
.../org/apache/hadoop/ozone/om/OzoneManager.java | 8 ++-
.../protocolPB/OzoneManagerRequestHandler.java | 3 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 58 ++++++++++++---
.../hadoop/ozone/client/ClientProtocolStub.java | 12 ++--
.../ozone/shell/snapshot/ListSnapshotHandler.java | 9 ++-
15 files changed, 320 insertions(+), 65 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 3a0a5cd8e8..d0eac8e881 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -551,14 +551,81 @@ public class ObjectStore {
/**
* List snapshots in a volume/bucket.
- * @param volumeName volume name
- * @param bucketName bucket name
- * @return list of snapshots for volume/bucket snapshotpath.
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @param snapshotPrefix snapshot prefix to match
+ * @param prevSnapshot snapshots will be listed after this snapshot name
* @throws IOException
*/
- public List<OzoneSnapshot> listSnapshot(String volumeName, String bucketName)
- throws IOException {
- return proxy.listSnapshot(volumeName, bucketName);
+ public Iterator<? extends OzoneSnapshot> listSnapshot(
+ String volumeName, String bucketName, String snapshotPrefix,
+ String prevSnapshot) throws IOException {
+ return new SnapshotIterator(
+ volumeName, bucketName, snapshotPrefix, prevSnapshot);
+ }
+
+ /**
+ * An Iterator to iterate over {@link OzoneSnapshot} list.
+ */
+ private class SnapshotIterator implements Iterator<OzoneSnapshot> {
+
+ private String volumeName = null;
+ private String bucketName = null;
+ private String snapshotPrefix = null;
+
+ private Iterator<OzoneSnapshot> currentIterator;
+ private OzoneSnapshot currentValue;
+
+ /**
+ * Creates an Iterator to iterate over all snapshots after
+ * prevSnapshot of specified bucket. If prevSnapshot is null it iterates
+ * from the first snapshot. The returned snapshots match snapshot prefix.
+ * @param snapshotPrefix snapshot prefix to match
+ * @param prevSnapshot snapshots will be listed after this snapshot name
+ */
+ SnapshotIterator(String volumeName, String bucketName,
+ String snapshotPrefix, String prevSnapshot) {
+ this.volumeName = volumeName;
+ this.bucketName = bucketName;
+ this.snapshotPrefix = snapshotPrefix;
+ this.currentValue = null;
+ this.currentIterator = getNextListOfSnapshots(prevSnapshot).iterator();
+ }
+
+ @Override
+ public boolean hasNext() {
+ // IMPORTANT: Without this logic, remote iteration will not work.
+ // Removing this will break the listSnapshot call if we try to
+ // list more than 1000 (ozone.client.list.cache ) snapshots.
+ if (!currentIterator.hasNext() && currentValue != null) {
+ currentIterator = getNextListOfSnapshots(currentValue.getName())
+ .iterator();
+ }
+ return currentIterator.hasNext();
+ }
+
+ @Override
+ public OzoneSnapshot next() {
+ if (hasNext()) {
+ currentValue = currentIterator.next();
+ return currentValue;
+ }
+ throw new NoSuchElementException();
+ }
+
+ /**
+ * Returns the next set of snapshot list using proxy.
+ * @param prevSnapshot previous snapshot, this will be excluded from result
+ * @return {@code List<OzoneSnapshot>}
+ */
+ private List<OzoneSnapshot> getNextListOfSnapshots(String prevSnapshot) {
+ try {
+ return proxy.listSnapshot(volumeName, bucketName, snapshotPrefix,
+ prevSnapshot, listCacheSize);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
}
public SnapshotDiffReport snapshotDiff(String volumeName, String bucketName,
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index b0400ac328..4ae176f94f 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -1042,13 +1042,17 @@ public interface ClientProtocol {
/**
* List snapshots in a volume/bucket.
- * @param volumeName volume name
- * @param bucketName bucket name
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @param snapshotPrefix snapshot prefix to match
+ * @param prevSnapshot start of the list, this snapshot is excluded
+ * @param maxListResult max numbet of snapshots to return
* @return list of snapshots for volume/bucket snapshotpath.
* @throws IOException
*/
- List<OzoneSnapshot> listSnapshot(String volumeName, String bucketName)
- throws IOException;
+ List<OzoneSnapshot> listSnapshot(
+ String volumeName, String bucketName, String snapshotPrefix,
+ String prevSnapshot, int maxListResult) throws IOException;
/**
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 4d1596c216..ee4c243310 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -985,19 +985,24 @@ public class RpcClient implements ClientProtocol {
/**
* List snapshots in a volume/bucket.
- * @param volumeName volume name
- * @param bucketName bucket name
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @param snapshotPrefix snapshot prefix to match
+ * @param prevSnapshot start of the list, this snapshot is excluded
+ * @param maxListResult max numbet of snapshots to return
* @return list of snapshots for volume/bucket snapshotpath.
* @throws IOException
*/
@Override
- public List<OzoneSnapshot> listSnapshot(String volumeName, String bucketName)
- throws IOException {
+ public List<OzoneSnapshot> listSnapshot(
+ String volumeName, String bucketName, String snapshotPrefix,
+ String prevSnapshot, int maxListResult) throws IOException {
Preconditions.checkArgument(Strings.isNotBlank(volumeName),
"volume can't be null or empty.");
Preconditions.checkArgument(Strings.isNotBlank(bucketName),
"bucket can't be null or empty.");
- return ozoneManagerClient.listSnapshot(volumeName, bucketName).stream()
+ return ozoneManagerClient.listSnapshot(volumeName, bucketName,
+ snapshotPrefix, prevSnapshot, maxListResult).stream()
.map(snapshotInfo -> OzoneSnapshot.fromSnapshotInfo(snapshotInfo))
.collect(Collectors.toList());
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index ed08f66013..bcab847986 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -664,13 +664,17 @@ public interface OzoneManagerProtocol
/**
* List snapshots in a volume/bucket.
- * @param volumeName volume name
- * @param bucketName bucket name
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @param snapshotPrefix snapshot prefix to match
+ * @param prevSnapshot start of the list, this snapshot is excluded
+ * @param maxListResult max numbet of snapshots to return
* @return list of snapshots for volume/bucket snapshotpath.
* @throws IOException
*/
- default List<SnapshotInfo> listSnapshot(String volumeName, String bucketName)
- throws IOException {
+ default List<SnapshotInfo> listSnapshot(
+ String volumeName, String bucketName, String snapshotPrefix,
+ String prevSnapshot, int maxListResult) throws IOException {
throw new UnsupportedOperationException("OzoneManager does not require " +
"this to be implemented");
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 998bca29d8..c20311d500 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -1153,13 +1153,23 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
* {@inheritDoc}
*/
@Override
- public List<SnapshotInfo> listSnapshot(String volumeName, String bucketName)
- throws IOException {
+ public List<SnapshotInfo> listSnapshot(
+ String volumeName, String bucketName, String snapshotPrefix,
+ String prevSnapshot, int maxListResult) throws IOException {
final OzoneManagerProtocolProtos.ListSnapshotRequest.Builder
requestBuilder =
OzoneManagerProtocolProtos.ListSnapshotRequest.newBuilder()
.setVolumeName(volumeName)
- .setBucketName(bucketName);
+ .setBucketName(bucketName)
+ .setMaxListResult(maxListResult);
+
+ if (prevSnapshot != null) {
+ requestBuilder.setPrevSnapshot(prevSnapshot);
+ }
+
+ if (snapshotPrefix != null) {
+ requestBuilder.setPrefix(snapshotPrefix);
+ }
final OMRequest omRequest = createOMRequest(Type.ListSnapshot)
.setListSnapshotRequest(requestBuilder)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index 029768bfda..92d3b2cee7 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
import org.apache.hadoop.ozone.client.OzoneKeyLocation;
import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts;
+import org.apache.hadoop.ozone.client.OzoneSnapshot;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.VolumeArgs;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
@@ -4117,4 +4118,86 @@ public abstract class TestOzoneRpcClientAbstract {
private static ReplicationConfig anyReplication() {
return
RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE);
}
+
+ @Test
+ public void testListSnapshot() throws IOException {
+ String volumeA = "vol-a-" + RandomStringUtils.randomNumeric(5);
+ String volumeB = "vol-b-" + RandomStringUtils.randomNumeric(5);
+ String bucketA = "buc-a-" + RandomStringUtils.randomNumeric(5);
+ String bucketB = "buc-b-" + RandomStringUtils.randomNumeric(5);
+ store.createVolume(volumeA);
+ store.createVolume(volumeB);
+ OzoneVolume volA = store.getVolume(volumeA);
+ OzoneVolume volB = store.getVolume(volumeB);
+ volA.createBucket(bucketA);
+ volA.createBucket(bucketB);
+ volB.createBucket(bucketA);
+ volB.createBucket(bucketB);
+ String snapshotPrefixA = "snapshot-a-";
+ String snapshotPrefixB = "snapshot-b-";
+ for (int i = 0; i < 10; i++) {
+ store.createSnapshot(volumeA, bucketA,
+ snapshotPrefixA + i + "-" + RandomStringUtils.randomNumeric(5));
+ store.createSnapshot(volumeA, bucketB,
+ snapshotPrefixA + i + "-" + RandomStringUtils.randomNumeric(5));
+ store.createSnapshot(volumeB, bucketA,
+ snapshotPrefixA + i + "-" + RandomStringUtils.randomNumeric(5));
+ store.createSnapshot(volumeB, bucketB,
+ snapshotPrefixA + i + "-" + RandomStringUtils.randomNumeric(5));
+ }
+ for (int i = 0; i < 10; i++) {
+ store.createSnapshot(volumeA, bucketA,
+ snapshotPrefixB + i + "-" + RandomStringUtils.randomNumeric(5));
+ store.createSnapshot(volumeA, bucketB,
+ snapshotPrefixB + i + "-" + RandomStringUtils.randomNumeric(5));
+ store.createSnapshot(volumeB, bucketA,
+ snapshotPrefixB + i + "-" + RandomStringUtils.randomNumeric(5));
+ store.createSnapshot(volumeB, bucketB,
+ snapshotPrefixB + i + "-" + RandomStringUtils.randomNumeric(5));
+ }
+
+ Iterator<? extends OzoneSnapshot> snapshotIter =
+ store.listSnapshot(volumeA, bucketA, null, null);
+ int volABucketASnapshotCount = 0;
+ while (snapshotIter.hasNext()) {
+ OzoneSnapshot snapshot = snapshotIter.next();
+ volABucketASnapshotCount++;
+ }
+ Assert.assertEquals(20, volABucketASnapshotCount);
+
+ snapshotIter = store.listSnapshot(volumeA, bucketB, null, null);
+ int volABucketBSnapshotCount = 0;
+ while (snapshotIter.hasNext()) {
+ OzoneSnapshot snapshot = snapshotIter.next();
+ volABucketBSnapshotCount++;
+ }
+ Assert.assertEquals(20, volABucketASnapshotCount);
+
+ snapshotIter = store.listSnapshot(volumeB, bucketA, null, null);
+ int volBBucketASnapshotCount = 0;
+ while (snapshotIter.hasNext()) {
+ OzoneSnapshot snapshot = snapshotIter.next();
+ volBBucketASnapshotCount++;
+ }
+ Assert.assertEquals(20, volABucketASnapshotCount);
+
+ snapshotIter = store.listSnapshot(volumeB, bucketB, null, null);
+ int volBBucketBSnapshotCount = 0;
+ while (snapshotIter.hasNext()) {
+ OzoneSnapshot snapshot = snapshotIter.next();
+ volBBucketBSnapshotCount++;
+ }
+ Assert.assertEquals(20, volABucketASnapshotCount);
+
+ int volABucketASnapshotACount = 0;
+ snapshotIter = store.listSnapshot(volumeA, bucketA, snapshotPrefixA, null);
+ while (snapshotIter.hasNext()) {
+ OzoneSnapshot snapshot = snapshotIter.next();
+ Assert.assertTrue(snapshot.getName().startsWith(snapshotPrefixA));
+ volABucketASnapshotACount++;
+ }
+ Assert.assertEquals(10, volABucketASnapshotACount);
+ Assert.assertFalse(snapshotIter.hasNext());
+
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
index 8bd09f543a..bd43b46b3f 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
@@ -430,7 +430,8 @@ public class TestOmMetrics {
writeClient.createSnapshot(volumeName, bucketName, snapshot2);
// List snapshots
- writeClient.listSnapshot(volumeName, bucketName);
+ writeClient.listSnapshot(
+ volumeName, bucketName, null, null, Integer.MAX_VALUE);
omMetrics = getMetrics("OMMetrics");
assertCounter("NumSnapshotActive", 2L, omMetrics);
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 81e776d6b4..b296acf587 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1674,6 +1674,9 @@ message CreateSnapshotRequest {
message ListSnapshotRequest {
optional string volumeName = 1;
optional string bucketName = 2;
+ optional string prefix = 3;
+ optional string prevSnapshot = 4;
+ optional uint32 maxListResult = 5;
}
message SnapshotDiffRequest {
diff --git
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
index 539a6daf6f..48948e3697 100644
---
a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
+++
b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java
@@ -224,12 +224,16 @@ public interface OMMetadataManager extends
DBStoreHAManager {
/**
* List snapshots in a volume/bucket.
- * @param volumeName volume name
- * @param bucketName bucket name
+ * @param volumeName volume name
+ * @param bucketName bucket name
+ * @param snapshotPrefix snapshot prefix to match
+ * @param prevSnapshot start of the list, this snapshot is excluded
+ * @param maxListResult max numbet of snapshots to return
* @return list of snapshot
*/
- List<SnapshotInfo> listSnapshot(String volumeName, String bucketName)
- throws IOException;
+ List<SnapshotInfo> listSnapshot(
+ String volumeName, String bucketName, String snapshotPrefix,
+ String prevSnapshot, int maxListResult) throws IOException;
/**
* Recover trash allows the user to recover the keys
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index cc5054c1aa..f9e5e35189 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -1088,7 +1088,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
if (StringUtil.isNotBlank(keyPrefix)) {
seekPrefix = getOzoneKey(volumeName, bucketName, keyPrefix);
} else {
- seekPrefix = getBucketKey(volumeName, bucketName + OM_KEY_PREFIX);
+ seekPrefix = getBucketKey(volumeName, bucketName) + OM_KEY_PREFIX;
}
int currentCount = 0;
@@ -1181,8 +1181,9 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
}
@Override
- public List<SnapshotInfo> listSnapshot(String volumeName, String bucketName)
- throws IOException {
+ public List<SnapshotInfo> listSnapshot(
+ String volumeName, String bucketName, String snapshotPrefix,
+ String prevSnapshot, int maxListResult) throws IOException {
if (Strings.isNullOrEmpty(volumeName)) {
throw new OMException("Volume name is required.", VOLUME_NOT_FOUND);
}
@@ -1197,39 +1198,69 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
BUCKET_NOT_FOUND);
}
- String prefix = getBucketKey(volumeName, bucketName + OM_KEY_PREFIX);
+ String prefix;
+ if (StringUtil.isNotBlank(snapshotPrefix)) {
+ prefix = getOzoneKey(volumeName, bucketName, snapshotPrefix);
+ } else {
+ prefix = getBucketKey(volumeName, bucketName + OM_KEY_PREFIX);
+ }
+
+ String seek;
+ if (StringUtil.isNotBlank(prevSnapshot)) {
+ // Seek to the specified snapshot.
+ seek = getOzoneKey(volumeName, bucketName, prevSnapshot);
+ } else {
+ // This allows us to seek directly to the first key with the right
prefix.
+ seek = getOzoneKey(volumeName, bucketName,
+ StringUtil.isNotBlank(
+ snapshotPrefix) ? snapshotPrefix : OM_KEY_PREFIX);
+ }
+
TreeMap<String, SnapshotInfo> snapshotInfoMap = new TreeMap<>();
- appendSnapshotFromCacheToMap(snapshotInfoMap, prefix);
- appendSnapshotFromDBToMap(snapshotInfoMap, prefix);
+ int count = appendSnapshotFromCacheToMap(
+ snapshotInfoMap, prefix, seek, maxListResult);
+ appendSnapshotFromDBToMap(
+ snapshotInfoMap, prefix, seek, count, maxListResult);
return new ArrayList<>(snapshotInfoMap.values());
}
- private void appendSnapshotFromCacheToMap(
- TreeMap snapshotInfoMap, String prefix) {
+ private int appendSnapshotFromCacheToMap(
+ TreeMap snapshotInfoMap, String prefix,
+ String previous, int maxListResult) {
+ int count = 0;
Iterator<Map.Entry<CacheKey<String>, CacheValue<SnapshotInfo>>> iterator =
snapshotInfoTable.cacheIterator();
- while (iterator.hasNext()) {
+ while (iterator.hasNext() && count < maxListResult) {
Map.Entry<CacheKey<String>, CacheValue<SnapshotInfo>> entry =
iterator.next();
String snapshotKey = entry.getKey().getCacheKey();
SnapshotInfo snapshotInfo = entry.getValue().getCacheValue();
- if (snapshotInfo != null && snapshotKey.startsWith(prefix)) {
+ if (snapshotInfo != null && snapshotKey.startsWith(prefix) &&
+ snapshotKey.compareTo(previous) > 0) {
snapshotInfoMap.put(snapshotKey, snapshotInfo);
+ count++;
}
}
+ return count;
}
- private void appendSnapshotFromDBToMap(TreeMap snapshotInfoMap, String
prefix)
+ private void appendSnapshotFromDBToMap(TreeMap snapshotInfoMap,
+ String prefix, String previous,
+ int count, int maxListResult)
throws IOException {
try (TableIterator<String, ? extends KeyValue<String, SnapshotInfo>>
snapshotIter = snapshotInfoTable.iterator()) {
KeyValue< String, SnapshotInfo> snapshotinfo;
- snapshotIter.seek(prefix);
- while (snapshotIter.hasNext()) {
+ snapshotIter.seek(previous);
+ while (snapshotIter.hasNext() && count < maxListResult) {
snapshotinfo = snapshotIter.next();
- if (snapshotinfo != null && snapshotinfo.getKey().startsWith(prefix)) {
+ if (snapshotinfo != null &&
+ snapshotinfo.getKey().compareTo(previous) == 0) {
+ continue;
+ }
+ if (snapshotinfo != null && snapshotinfo.getKey().startsWith(prefix))
{
CacheValue<SnapshotInfo> cacheValue =
snapshotInfoTable.getCacheValue(
new CacheKey<>(snapshotinfo.getKey()));
@@ -1238,6 +1269,7 @@ public class OmMetadataManagerImpl implements
OMMetadataManager {
// in cache.
if (cacheValue == null) {
snapshotInfoMap.put(snapshotinfo.getKey(),
snapshotinfo.getValue());
+ count++;
}
} else {
break;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 169e87fa39..52fa7ac9b1 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -2777,8 +2777,9 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
}
@Override
- public List<SnapshotInfo> listSnapshot(String volumeName, String bucketName)
- throws IOException {
+ public List<SnapshotInfo> listSnapshot(
+ String volumeName, String bucketName, String snapshotPrefix,
+ String prevSnapshot, int maxListResult) throws IOException {
if (isAclEnabled) {
omMetadataReader.checkAcls(ResourceType.BUCKET, StoreType.OZONE,
ACLType.LIST, volumeName, bucketName, null);
@@ -2788,7 +2789,8 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
auditMap.put(OzoneConsts.BUCKET, bucketName);
try {
metrics.incNumSnapshotLists();
- return metadataManager.listSnapshot(volumeName, bucketName);
+ return metadataManager.listSnapshot(volumeName, bucketName,
+ snapshotPrefix, prevSnapshot, maxListResult);
} catch (Exception ex) {
metrics.incNumSnapshotListFails();
auditSuccess = false;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 6ab84a5f02..31af0f16e1 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -1243,7 +1243,8 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
OzoneManagerProtocolProtos.ListSnapshotRequest request)
throws IOException {
List<SnapshotInfo> snapshotInfos = impl.listSnapshot(
- request.getVolumeName(), request.getBucketName());
+ request.getVolumeName(), request.getBucketName(), request.getPrefix(),
+ request.getPrevSnapshot(), request.getMaxListResult());
List<OzoneManagerProtocolProtos.SnapshotInfo> snapshotInfoList =
snapshotInfos.stream().map(SnapshotInfo::getProtobuf)
.collect(Collectors.toList());
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
index 1ff17bce38..8d3e5ddc0a 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -686,26 +686,64 @@ public class TestOmMetadataManager {
OMRequestTestUtils.addVolumeToDB(vol1, omMetadataManager);
addBucketsToCache(vol1, bucket1);
- String snapshotName = "snapshot";
+ String prefixA = "snapshotA";
+ String prefixB = "snapshotB";
+ TreeSet<String> snapshotsASet = new TreeSet<>();
- for (int i = 1; i <= 10; i++) {
+ for (int i = 1; i <= 100; i++) {
if (i % 2 == 0) {
+ snapshotsASet.add(prefixA + i);
OMRequestTestUtils.addSnapshotToTable(vol1, bucket1,
- snapshotName + i, omMetadataManager);
+ prefixA + i, omMetadataManager);
} else {
OMRequestTestUtils.addSnapshotToTableCache(vol1, bucket1,
- snapshotName + i, omMetadataManager);
+ prefixB + i, omMetadataManager);
}
}
//Test listing all snapshots.
List<SnapshotInfo> snapshotInfos = omMetadataManager.listSnapshot(vol1,
- bucket1);
- assertEquals(10, snapshotInfos.size());
+ bucket1, null, null, 100);
+ assertEquals(100, snapshotInfos.size());
+
+ snapshotInfos = omMetadataManager.listSnapshot(vol1,
+ bucket1, prefixA, null, 50);
+ assertEquals(50, snapshotInfos.size());
+ for (SnapshotInfo snapshotInfo : snapshotInfos) {
+ assertTrue(snapshotInfo.getName().startsWith(prefixA));
+ }
+
+ String startSnapshot = prefixA + 38;
+ snapshotInfos = omMetadataManager.listSnapshot(vol1,
+ bucket1, prefixA, startSnapshot, 50);
+ assertEquals(snapshotsASet.tailSet(startSnapshot).size() - 1,
+ snapshotInfos.size());
for (SnapshotInfo snapshotInfo : snapshotInfos) {
- assertTrue(snapshotInfo.getName().startsWith(snapshotName));
+ assertTrue(snapshotInfo.getName().startsWith(prefixA));
+ assertTrue(snapshotInfo.getName().compareTo(startSnapshot) > 0);
}
+ startSnapshot = null;
+ TreeSet<String> expectedSnapshot = new TreeSet<>();
+ for (int i = 1; i <= 5; i++) {
+ snapshotInfos = omMetadataManager.listSnapshot(
+ vol1, bucket1, prefixA, startSnapshot, 10);
+ assertEquals(10, snapshotInfos.size());
+
+ for (SnapshotInfo snapshotInfo : snapshotInfos) {
+ expectedSnapshot.add(snapshotInfo.getName());
+ assertTrue(snapshotInfo.getName().startsWith(prefixA));
+ startSnapshot = snapshotInfo.getName();
+ }
+ }
+ assertEquals(snapshotsASet, expectedSnapshot);
+
+ // As now we have iterated all 50 snapshots, calling next time should
+ // return empty list.
+ snapshotInfos = omMetadataManager.listSnapshot(vol1, bucket1,
+ startSnapshot, prefixA, 10);
+
+ assertEquals(snapshotInfos.size(), 0);
}
@ParameterizedTest
@@ -721,7 +759,7 @@ public class TestOmMetadataManager {
addBucketsToCache(vol1, bucket1);
OMException oe = assertThrows(OMException.class,
- () -> omMetadataManager.listSnapshot(volume, bucket));
+ () -> omMetadataManager.listSnapshot(volume, bucket, null, null, 100));
assertEquals(expectedResultCode, oe.getResult());
}
@@ -758,14 +796,14 @@ public class TestOmMetadataManager {
//Test listing snapshots only lists snapshots of specified bucket
List<SnapshotInfo> snapshotInfos1 = omMetadataManager.listSnapshot(vol1,
- bucket1);
+ bucket1, null, null, Integer.MAX_VALUE);
assertEquals(2, snapshotInfos1.size());
for (SnapshotInfo snapshotInfo : snapshotInfos1) {
assertTrue(snapshotInfo.getName().startsWith(snapshotName1));
}
List<SnapshotInfo> snapshotInfos2 = omMetadataManager.listSnapshot(vol1,
- bucket2);
+ bucket2, null, null, Integer.MAX_VALUE);
assertEquals(5, snapshotInfos2.size());
for (SnapshotInfo snapshotInfo : snapshotInfos2) {
assertTrue(snapshotInfo.getName().startsWith(snapshotName2));
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index aa1cb345df..3bb232f338 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -617,17 +617,19 @@ public class ClientProtocolStub implements ClientProtocol
{
}
@Override
+ public List<OzoneSnapshot> listSnapshot(
+ String volumeName, String bucketName, String snapshotPrefix,
+ String prevSnapshot, int maxListResult) throws IOException {
+ return null;
+ }
+
public void deleteSnapshot(String volumeName,
String bucketName, String snapshotName)
throws IOException {
}
- @Override
- public List<OzoneSnapshot> listSnapshot(String volumeName, String bucketName)
- throws IOException {
- return null;
- }
+
public SnapshotDiffReport snapshotDiff(String volumeName, String bucketName,
String fromSnapshot, String
toSnapshot)
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
index 7ad4041bef..8ac9471f26 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/ListSnapshotHandler.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.ozone.shell.bucket.BucketUri;
import picocli.CommandLine;
import java.io.IOException;
-import java.util.List;
+import java.util.Iterator;
/**
* ozone sh snapshot list.
@@ -50,10 +50,9 @@ public class ListSnapshotHandler extends Handler {
String volumeName = snapshotPath.getValue().getVolumeName();
String bucketName = snapshotPath.getValue().getBucketName();
- List<? extends OzoneSnapshot> snapshotInfos = client.getObjectStore()
- .listSnapshot(volumeName, bucketName);
- int counter = printAsJsonArray(snapshotInfos.iterator(),
- snapshotInfos.size());
+ Iterator<? extends OzoneSnapshot> snapshotInfos = client.getObjectStore()
+ .listSnapshot(volumeName, bucketName, null, null);
+ int counter = printAsJsonArray(snapshotInfos, Integer.MAX_VALUE);
if (isVerbose()) {
out().printf("Found : %d snapshots for o3://%s/ %s ", counter,
volumeName, bucketName);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
