This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new c168fa06fd HDDS-7950. [Snapshot] API changes to support pagination
token and async nature of snapshot diff API (#4264)
c168fa06fd is described below
commit c168fa06fdf8204b91958ba017d4bc4c2830f052
Author: Hemant Kumar <[email protected]>
AuthorDate: Mon Mar 6 12:39:13 2023 -0800
HDDS-7950. [Snapshot] API changes to support pagination token and async
nature of snapshot diff API (#4264)
---
.../apache/hadoop/ozone/client/ObjectStore.java | 26 +++++--
.../ozone/client/protocol/ClientProtocol.java | 9 ++-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 12 ++-
.../ozone/om/protocol/OzoneManagerProtocol.java | 14 ++--
...OzoneManagerProtocolClientSideTranslatorPB.java | 27 +++++--
.../hadoop/ozone/snapshot/SnapshotDiffReport.java | 40 +++++++---
.../ozone/snapshot/SnapshotDiffResponse.java | 85 ++++++++++++++++++++++
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 51 +++++++++----
.../src/main/proto/OmClientProtocol.proto | 12 +++
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 19 +++--
.../org/apache/hadoop/ozone/om/OzoneManager.java | 12 ++-
.../protocolPB/OzoneManagerRequestHandler.java | 20 ++++-
.../hadoop/ozone/client/ClientProtocolStub.java | 11 ++-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 12 ++-
14 files changed, 285 insertions(+), 65 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 3a0a5cd8e8..30e13b2f65 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue;
import org.apache.hadoop.ozone.om.helpers.TenantUserList;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.security.UserGroupInformation;
import com.google.common.annotations.VisibleForTesting;
@@ -553,7 +553,7 @@ public class ObjectStore {
* List snapshots in a volume/bucket.
* @param volumeName volume name
* @param bucketName bucket name
- * @return list of snapshots for volume/bucket snapshotpath.
+ * @return list of snapshots for volume/bucket snapshot path.
* @throws IOException
*/
public List<OzoneSnapshot> listSnapshot(String volumeName, String bucketName)
@@ -561,9 +561,25 @@ public class ObjectStore {
return proxy.listSnapshot(volumeName, bucketName);
}
- public SnapshotDiffReport snapshotDiff(String volumeName, String bucketName,
- String fromSnapshot, String
toSnapshot)
+ /**
+ * Get the differences between two snapshots.
+ * @param volumeName Name of the volume to which the snapshot bucket belong
+ * @param bucketName Name of the bucket to which the snapshots belong
+ * @param fromSnapshot The name of the starting snapshot
+ * @param toSnapshot The name of the ending snapshot
+ * @param token to get the index to return diff report from.
+ * @param pageSize maximum entries returned to the report.
+ * @return the difference report between two snapshots
+ * @throws IOException in case of any exception while generating snapshot
diff
+ */
+ public SnapshotDiffResponse snapshotDiff(String volumeName,
+ String bucketName,
+ String fromSnapshot,
+ String toSnapshot,
+ String token,
+ int pageSize)
throws IOException {
- return proxy.snapshotDiff(volumeName, bucketName, fromSnapshot,
toSnapshot);
+ return proxy.snapshotDiff(volumeName, bucketName, fromSnapshot, toSnapshot,
+ token, pageSize);
}
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index b0400ac328..3e4abd62ef 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -63,7 +63,7 @@ import org.apache.hadoop.ozone.om.protocol.S3Auth;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRoleInfo;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.token.Token;
@@ -1057,10 +1057,13 @@ public interface ClientProtocol {
* @param bucketName Name of the bucket to which the snapshots belong
* @param fromSnapshot The name of the starting snapshot
* @param toSnapshot The name of the ending snapshot
+ * @param token to get the index to return diff report from.
+ * @param pageSize maximum entries returned to the report.
* @return the difference report between two snapshots
* @throws IOException in case of any exception while generating snapshot
diff
*/
- SnapshotDiffReport snapshotDiff(String volumeName, String bucketName,
- String fromSnapshot, String toSnapshot)
+ SnapshotDiffResponse snapshotDiff(String volumeName, String bucketName,
+ String fromSnapshot, String toSnapshot,
+ String token, int pageSize)
throws IOException;
}
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 128ad4fe68..7676bc6dc4 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -151,7 +151,7 @@ import
org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
import org.apache.hadoop.ozone.security.acl.OzoneAclConfig;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -979,15 +979,19 @@ public class RpcClient implements ClientProtocol {
}
@Override
- public SnapshotDiffReport snapshotDiff(String volumeName, String bucketName,
- String fromSnapshot, String
toSnapshot)
+ public SnapshotDiffResponse snapshotDiff(String volumeName,
+ String bucketName,
+ String fromSnapshot,
+ String toSnapshot,
+ String token,
+ int pageSize)
throws IOException {
Preconditions.checkArgument(Strings.isNotBlank(volumeName),
"volume can't be null or empty.");
Preconditions.checkArgument(Strings.isNotBlank(bucketName),
"bucket can't be null or empty.");
return ozoneManagerClient.snapshotDiff(volumeName, bucketName,
- fromSnapshot, toSnapshot);
+ fromSnapshot, toSnapshot, token, pageSize);
}
/**
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 5c1fa78a4d..19331b472c 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -64,7 +64,7 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CancelP
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.EchoRPCResponse;
import org.apache.hadoop.ozone.security.OzoneDelegationTokenSelector;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.token.TokenInfo;
@@ -703,13 +703,17 @@ public interface OzoneManagerProtocol
* @param bucketName Name of the bucket to which the snapshots belong
* @param fromSnapshot The name of the starting snapshot
* @param toSnapshot The name of the ending snapshot
+ * @param token to get the index to return diff report from.
+ * @param pageSize maximum entries returned to the report.
* @return the difference report between two snapshots
* @throws IOException in case of any exception while generating snapshot
diff
*/
- default SnapshotDiffReport snapshotDiff(String volumeName,
- String bucketName,
- String fromSnapshot,
- String toSnapshot)
+ default SnapshotDiffResponse snapshotDiff(String volumeName,
+ String bucketName,
+ String fromSnapshot,
+ String toSnapshot,
+ String token,
+ int pageSize)
throws IOException {
throw new UnsupportedOperationException("OzoneManager does not require " +
"this to be implemented");
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 0820e4a280..d01d0d3ee7 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -190,6 +190,8 @@ import
org.apache.hadoop.ozone.security.proto.SecurityProtos.CancelDelegationTok
import
org.apache.hadoop.ozone.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
import
org.apache.hadoop.ozone.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer;
import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages;
import org.apache.hadoop.security.token.Token;
@@ -1195,24 +1197,37 @@ public final class
OzoneManagerProtocolClientSideTranslatorPB
* {@inheritDoc}
*/
@Override
- public SnapshotDiffReport snapshotDiff(String volumeName, String bucketName,
- String fromSnapshot, String
toSnapshot)
- throws IOException {
+ public SnapshotDiffResponse snapshotDiff(String volumeName,
+ String bucketName,
+ String fromSnapshot,
+ String toSnapshot,
+ String token,
+ int pageSize) throws IOException {
final OzoneManagerProtocolProtos.SnapshotDiffRequest.Builder
requestBuilder =
OzoneManagerProtocolProtos.SnapshotDiffRequest.newBuilder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setFromSnapshot(fromSnapshot)
- .setToSnapshot(toSnapshot);
+ .setToSnapshot(toSnapshot)
+ .setPageSize(pageSize);
+
+ if (!StringUtils.isBlank(token)) {
+ requestBuilder.setToken(token);
+ }
final OMRequest omRequest = createOMRequest(Type.SnapshotDiff)
.setSnapshotDiffRequest(requestBuilder)
.build();
final OMResponse omResponse = submitRequest(omRequest);
handleError(omResponse);
- return SnapshotDiffReport.fromProtobuf(omResponse
- .getSnapshotDiffResponse().getSnapshotDiffReport());
+ OzoneManagerProtocolProtos.SnapshotDiffResponse diffResponse =
+ omResponse.getSnapshotDiffResponse();
+
+ return new SnapshotDiffResponse(
+ SnapshotDiffReport.fromProtobuf(diffResponse.getSnapshotDiffReport()),
+ JobStatus.fromProtobuf(diffResponse.getJobStatus()),
+ diffResponse.getWaitTimeInMs());
}
/**
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java
index e8e387099c..83b52db6bf 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffReport.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.snapshot;
+import org.apache.commons.lang3.StringUtils;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffReportProto;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DiffReportEntryProto;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DiffReportEntryProto.DiffTypeProto;
@@ -182,9 +183,17 @@ public class SnapshotDiffReport {
*/
private final List<DiffReportEntry> diffList;
- public SnapshotDiffReport(final String volumeName, final String bucketName,
- final String fromSnapshot, final String toSnapshot,
- List<DiffReportEntry> entryList) {
+ /**
+ * subsequent token for the diff report.
+ */
+ // TODO: [SNAPSHOT] will set it properly in HDDS-7548
+ private final String token = null;
+
+ public SnapshotDiffReport(final String volumeName,
+ final String bucketName,
+ final String fromSnapshot,
+ final String toSnapshot,
+ final List<DiffReportEntry> entryList) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.fromSnapshot = fromSnapshot;
@@ -199,14 +208,19 @@ public class SnapshotDiffReport {
@Override
public String toString() {
StringBuilder str = new StringBuilder();
- String from = "snapshot " + fromSnapshot;
- String to = "snapshot " + toSnapshot;
- str.append("Difference between ").append(from).append(" and ").append(to)
- .append(":")
+ str.append("Difference between snapshot: ")
+ .append(fromSnapshot)
+ .append(" and snapshot: ")
+ .append(toSnapshot)
.append(LINE_SEPARATOR);
for (DiffReportEntry entry : diffList) {
str.append(entry.toString()).append(LINE_SEPARATOR);
}
+ if (StringUtils.isNotEmpty(token)) {
+ str.append("Next token: ")
+ .append(token)
+ .append(LINE_SEPARATOR);
+ }
return str.toString();
}
@@ -219,15 +233,21 @@ public class SnapshotDiffReport {
.setToSnapshot(toSnapshot);
builder.addAllDiffList(diffList.stream().map(DiffReportEntry::toProtobuf)
.collect(Collectors.toList()));
+ if (StringUtils.isNotEmpty(token)) {
+ builder.setToken(token);
+ }
return builder.build();
}
public static SnapshotDiffReport fromProtobuf(
final SnapshotDiffReportProto report) {
return new SnapshotDiffReport(report.getVolumeName(),
- report.getBucketName(), report.getFromSnapshot(),
- report.getToSnapshot(), report.getDiffListList().stream()
- .map(DiffReportEntry::fromProtobuf).collect(Collectors.toList()));
+ report.getBucketName(),
+ report.getFromSnapshot(),
+ report.getToSnapshot(),
+ report.getDiffListList().stream()
+ .map(DiffReportEntry::fromProtobuf)
+ .collect(Collectors.toList()));
}
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java
new file mode 100644
index 0000000000..8175ae86eb
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/snapshot/SnapshotDiffResponse.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.snapshot;
+
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SnapshotDiffResponse.JobStatusProto;
+
+/**
+ * POJO for Snapshot Diff Response.
+ */
+public class SnapshotDiffResponse {
+
+ /**
+ * Snapshot diff job status enum.
+ */
+ public enum JobStatus {
+ QUEUED,
+ IN_PROGRESS,
+ DONE,
+ REJECTED,
+ FAILED;
+
+ public JobStatusProto toProtobuf() {
+ return JobStatusProto.valueOf(this.name());
+ }
+
+ public static JobStatus fromProtobuf(JobStatusProto jobStatusProto) {
+ return JobStatus.valueOf(jobStatusProto.name());
+ }
+ }
+
+ private final SnapshotDiffReport snapshotDiffReport;
+ private final JobStatus jobStatus;
+ private final long waitTimeInMs;
+
+ public SnapshotDiffResponse(final SnapshotDiffReport snapshotDiffReport,
+ final JobStatus jobStatus,
+ final long waitTimeInMs) {
+ this.snapshotDiffReport = snapshotDiffReport;
+ this.jobStatus = jobStatus;
+ this.waitTimeInMs = waitTimeInMs;
+ }
+
+ public SnapshotDiffReport getSnapshotDiffReport() {
+ return snapshotDiffReport;
+ }
+
+ public JobStatus getJobStatus() {
+ return jobStatus;
+ }
+
+ public long getWaitTimeInMs() {
+ return waitTimeInMs;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder str = new StringBuilder();
+ if (jobStatus == JobStatus.DONE) {
+ str.append(snapshotDiffReport.toString());
+ } else {
+ str.append("Snapshot diff job is ");
+ str.append(jobStatus);
+ str.append("\n");
+ str.append("Please retry after ");
+ str.append(waitTimeInMs);
+ str.append(" ms.");
+ }
+ return str.toString();
+ }
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index a5941fff9f..8b70349f4b 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ozone.test.LambdaTestUtils;
import org.jetbrains.annotations.NotNull;
@@ -78,6 +79,7 @@ import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.CONT
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static
org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
+import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.junit.Assert.assertThrows;
@@ -486,7 +488,8 @@ public class TestOmSnapshot {
// Do nothing, take another snapshot
String snap2 = "snap" + RandomStringUtils.randomNumeric(5);
createSnapshot(volume, bucket, snap2);
- SnapshotDiffReport diff1 = store.snapshotDiff(volume, bucket, snap1,
snap2);
+
+ SnapshotDiffReport diff1 = getSnapDiffReport(volume, bucket, snap1, snap2);
Assert.assertTrue(diff1.getDiffList().isEmpty());
// Create Key2 and delete Key1, take snapshot
String key2 = "key-2-";
@@ -494,8 +497,9 @@ public class TestOmSnapshot {
bucket1.deleteKey(key1);
String snap3 = "snap" + RandomStringUtils.randomNumeric(5);
createSnapshot(volume, bucket, snap3);
+
// Diff should have 2 entries
- SnapshotDiffReport diff2 = store.snapshotDiff(volume, bucket, snap2,
snap3);
+ SnapshotDiffReport diff2 = getSnapDiffReport(volume, bucket, snap2, snap3);
Assert.assertEquals(2, diff2.getDiffList().size());
Assert.assertTrue(diff2.getDiffList().contains(
SnapshotDiffReport.DiffReportEntry
@@ -509,7 +513,8 @@ public class TestOmSnapshot {
bucket1.renameKey(key2, key2Renamed);
String snap4 = "snap" + RandomStringUtils.randomNumeric(5);
createSnapshot(volume, bucket, snap4);
- SnapshotDiffReport diff3 = store.snapshotDiff(volume, bucket, snap3,
snap4);
+
+ SnapshotDiffReport diff3 = getSnapDiffReport(volume, bucket, snap3, snap4);
Assert.assertEquals(1, diff3.getDiffList().size());
Assert.assertTrue(diff3.getDiffList().contains(
SnapshotDiffReport.DiffReportEntry
@@ -521,7 +526,7 @@ public class TestOmSnapshot {
bucket1.createDirectory(dir1);
String snap5 = "snap" + RandomStringUtils.randomNumeric(5);
createSnapshot(volume, bucket, snap5);
- SnapshotDiffReport diff4 = store.snapshotDiff(volume, bucket, snap4,
snap5);
+ SnapshotDiffReport diff4 = getSnapDiffReport(volume, bucket, snap4, snap5);
Assert.assertEquals(1, diff4.getDiffList().size());
// for non-fso, directories are a special type of key with "/" appended
// at the end.
@@ -534,6 +539,21 @@ public class TestOmSnapshot {
}
+ private SnapshotDiffReport getSnapDiffReport(String volume,
+ String bucket,
+ String fromSnapshot,
+ String toSnapshot)
+ throws InterruptedException, IOException {
+ SnapshotDiffResponse response;
+ do {
+ response = store.snapshotDiff(volume, bucket, fromSnapshot,
+ toSnapshot, null, 0);
+ Thread.sleep(response.getWaitTimeInMs());
+ } while (response.getJobStatus() != DONE);
+
+ return response.getSnapshotDiffReport();
+ }
+
@Test
public void testSnapDiffNoSnapshot() throws Exception {
String volume = "vol-" + RandomStringUtils.randomNumeric(5);
@@ -551,11 +571,11 @@ public class TestOmSnapshot {
// Destination snapshot is invalid
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volume, bucket, snap1, snap2));
+ () -> store.snapshotDiff(volume, bucket, snap1, snap2, null, 0));
// From snapshot is invalid
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volume, bucket, snap2, snap1));
+ () -> store.snapshotDiff(volume, bucket, snap2, snap1, null, 0));
}
@Test
@@ -580,15 +600,15 @@ public class TestOmSnapshot {
// Bucket is nonexistent
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volumea, bucketb, snap1, snap2));
+ () -> store.snapshotDiff(volumea, bucketb, snap1, snap2, null, 0));
// Volume is nonexistent
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volumeb, bucketa, snap2, snap1));
+ () -> store.snapshotDiff(volumeb, bucketa, snap2, snap1, null, 0));
// Both volume and bucket are nonexistent
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volumeb, bucketb, snap2, snap1));
+ () -> store.snapshotDiff(volumeb, bucketb, snap2, snap1, null, 0));
}
@Test
@@ -610,17 +630,17 @@ public class TestOmSnapshot {
// Destination snapshot is empty
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volume, bucket, snap1, nullstr));
+ () -> store.snapshotDiff(volume, bucket, snap1, nullstr, null, 0));
// From snapshot is empty
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volume, bucket, nullstr, snap1));
+ () -> store.snapshotDiff(volume, bucket, nullstr, snap1, null, 0));
// Bucket is empty
assertThrows(IllegalArgumentException.class,
- () -> store.snapshotDiff(volume, nullstr, snap1, snap2));
+ () -> store.snapshotDiff(volume, nullstr, snap1, snap2, null, 0));
// Volume is empty
assertThrows(IllegalArgumentException.class,
- () -> store.snapshotDiff(nullstr, bucket, snap1, snap2));
+ () -> store.snapshotDiff(nullstr, bucket, snap1, snap2, null, 0));
}
@Test
@@ -646,7 +666,7 @@ public class TestOmSnapshot {
String snap2 = "snap" + RandomStringUtils.randomNumeric(5);
createSnapshot(volume, bucketName1, snap2);
SnapshotDiffReport diff1 =
- store.snapshotDiff(volume, bucketName1, snap1, snap2);
+ getSnapDiffReport(volume, bucketName1, snap1, snap2);
Assert.assertEquals(1, diff1.getDiffList().size());
}
@@ -693,7 +713,8 @@ public class TestOmSnapshot {
createSnapshot(volumeName1, bucketName1, snap2); // 1.sst 2.sst 3.sst 4.sst
Assert.assertEquals(4, getKeyTableSstFiles().size());
SnapshotDiffReport diff1 =
- store.snapshotDiff(volumeName1, bucketName1, snap1, snap2);
+ store.snapshotDiff(volumeName1, bucketName1, snap1, snap2, null, 0)
+ .getSnapshotDiffReport();
Assert.assertEquals(1, diff1.getDiffList().size());
}
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 33ee575d6e..5f028e939f 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1692,6 +1692,8 @@ message SnapshotDiffRequest {
optional string bucketName = 2;
optional string fromSnapshot = 3;
optional string toSnapshot = 4;
+ optional string token = 5;
+ optional uint32 pageSize = 6;
}
message DeleteSnapshotRequest {
@@ -1747,7 +1749,16 @@ message ListSnapshotResponse {
}
message SnapshotDiffResponse {
+ enum JobStatusProto {
+ QUEUED = 1;
+ IN_PROGRESS = 2;
+ DONE = 3;
+ REJECTED = 4;
+ FAILED = 5;
+ }
optional SnapshotDiffReportProto snapshotDiffReport = 1;
+ optional JobStatusProto jobStatus = 2;
+ optional int64 waitTimeInMs = 3;
}
message DeleteSnapshotResponse {
@@ -1760,6 +1771,7 @@ message SnapshotDiffReportProto {
optional string fromSnapshot = 3;
optional string toSnapshot = 4;
repeated DiffReportEntryProto diffList = 5;
+ optional string token = 6;
}
message DiffReportEntryProto {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 1c1c683c90..bbb784cfe4 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus;
import org.apache.hadoop.ozone.om.snapshot.SnapshotDiffManager;
import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer;
import org.rocksdb.RocksDBException;
import org.slf4j.Logger;
@@ -54,6 +55,7 @@ import static
org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DB_DIR;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_KEY_NAME;
import static
org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
+import static
org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
/**
* This class is used to manage/create OM snapshots.
@@ -254,10 +256,14 @@ public final class OmSnapshotManager implements
AutoCloseable {
(keyParts[0].compareTo(OM_SNAPSHOT_INDICATOR) == 0);
}
- public SnapshotDiffReport getSnapshotDiffReport(final String volume,
- final String bucket,
- final String fromSnapshot,
- final String toSnapshot)
+ // TODO: [SNAPSHOT] Will pass token and page size to snapshotDiffManager in
+ // HDDS-7548
+ public SnapshotDiffResponse getSnapshotDiffReport(final String volume,
+ final String bucket,
+ final String fromSnapshot,
+ final String toSnapshot,
+ final String token,
+ final int pageSize)
throws IOException {
// Validate fromSnapshot and toSnapshot
final SnapshotInfo fsInfo = getSnapshotInfo(volume, bucket, fromSnapshot);
@@ -269,8 +275,9 @@ public final class OmSnapshotManager implements
AutoCloseable {
try {
final OmSnapshot fs = snapshotCache.get(fsKey);
final OmSnapshot ts = snapshotCache.get(tsKey);
- return snapshotDiffManager.getSnapshotDiffReport(volume, bucket, fs, ts,
- fsInfo, tsInfo);
+ SnapshotDiffReport snapshotDiffReport = snapshotDiffManager
+ .getSnapshotDiffReport(volume, bucket, fs, ts, fsInfo, tsInfo);
+ return new SnapshotDiffResponse(snapshotDiffReport, DONE, 0L);
} catch (ExecutionException | RocksDBException e) {
throw new IOException(e.getCause());
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 5cdd94b1f1..6c45d4444e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -88,7 +88,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
import org.apache.hadoop.ozone.om.service.OMRangerBGSyncService;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.ozone.util.OzoneNetUtils;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo;
@@ -4471,11 +4471,15 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
ozoneObj.getKeyName());
}
- public SnapshotDiffReport snapshotDiff(String volume, String bucket,
- String fromSnapshot, String
toSnapshot)
+ public SnapshotDiffResponse snapshotDiff(String volume,
+ String bucket,
+ String fromSnapshot,
+ String toSnapshot,
+ String token,
+ int pageSize)
throws IOException {
return omSnapshotManager.getSnapshotDiffReport(volume, bucket,
- fromSnapshot, toSnapshot);
+ fromSnapshot, toSnapshot, token, pageSize);
}
@Override // ReconfigureProtocol
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 1032de2542..973953d76f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -1219,11 +1219,25 @@ public class OzoneManagerRequestHandler implements
RequestHandler {
private SnapshotDiffResponse snapshotDiff(
SnapshotDiffRequest snapshotDiffRequest) throws IOException {
- return SnapshotDiffResponse.newBuilder().setSnapshotDiffReport(
- impl.snapshotDiff(snapshotDiffRequest.getVolumeName(),
+ org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse response =
+ impl.snapshotDiff(
+ snapshotDiffRequest.getVolumeName(),
snapshotDiffRequest.getBucketName(),
snapshotDiffRequest.getFromSnapshot(),
- snapshotDiffRequest.getToSnapshot()).toProtobuf()).build();
+ snapshotDiffRequest.getToSnapshot(),
+ snapshotDiffRequest.getToken(),
+ snapshotDiffRequest.getPageSize());
+
+ SnapshotDiffResponse.Builder builder = SnapshotDiffResponse.newBuilder()
+ .setJobStatus(response.getJobStatus().toProtobuf())
+ .setWaitTimeInMs(response.getWaitTimeInMs());
+
+ if (response.getSnapshotDiffReport() != null) {
+ builder.setSnapshotDiffReport(
+ response.getSnapshotDiffReport().toProtobuf());
+ }
+
+ return builder.build();
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index aa1cb345df..9495d2869c 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -48,7 +48,7 @@ import org.apache.hadoop.ozone.om.protocol.S3Auth;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.security.OzoneTokenIdentifier;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
-import org.apache.hadoop.ozone.snapshot.SnapshotDiffReport;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
import org.apache.hadoop.security.token.Token;
import java.io.IOException;
@@ -629,8 +629,13 @@ public class ClientProtocolStub implements ClientProtocol {
return null;
}
- public SnapshotDiffReport snapshotDiff(String volumeName, String bucketName,
- String fromSnapshot, String
toSnapshot)
+ @Override
+ public SnapshotDiffResponse snapshotDiff(String volumeName,
+ String bucketName,
+ String fromSnapshot,
+ String toSnapshot,
+ String token,
+ int pageSize)
throws IOException {
return null;
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java
index 508bda5cc7..7690bd054c 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java
@@ -45,6 +45,15 @@ public class SnapshotDiffHandler extends Handler {
index = "2")
private String toSnapshot;
+ @CommandLine.Option(names = {"-t", "--token"},
+ description = "continuation token for next page (optional)")
+ private String token;
+
+ @CommandLine.Option(names = {"-p", "--page-size"},
+ description = "number of diff entries to be returned in the response" +
+ " (optional)")
+ private int pageSize;
+
@Override
protected OzoneAddress getAddress() {
return snapshotPath.getValue();
@@ -61,7 +70,8 @@ public class SnapshotDiffHandler extends Handler {
try (PrintStream stream = out()) {
stream.print(client.getObjectStore()
- .snapshotDiff(volumeName, bucketName, fromSnapshot, toSnapshot));
+ .snapshotDiff(volumeName, bucketName, fromSnapshot, toSnapshot,
+ token, pageSize));
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
