smengcl commented on code in PR #11651:
URL: https://github.com/apache/ozone/pull/11651#discussion_r4394037568
##########
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java:
##########
@@ -1588,11 +1606,190 @@ public void testSubmitSnapshotDiffDoneShortCircuits()
SnapshotDiffManager spy = spy(snapshotDiffManager);
SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName,
- ctx.bucketName, ctx.fromSnapshotName, ctx.toSnapshotName, false,
false);
+ ctx.bucketName, ctx.fromSnapshotName, ctx.toSnapshotName, false,
false, false);
assertNotNull(response);
verify(spy, never()).generateSnapshotDiffReport(anyString(), anyString(),
- anyString(), anyString(), anyString(), anyString(), anyBoolean(),
anyBoolean());
+ anyString(), anyString(), anyString(), anyString(), anyBoolean(),
anyBoolean(),
+ anyBoolean());
+ }
+
+ @Test
+ public void testSubmitDoneResubmitsFsoOrdered()
+ throws Exception {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ DONE, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 1L, null, 0.0, null);
+ SnapshotDiffManager optimizedManager =
newSnapshotDiffManagerWithOptimizedEnabled(true);
+ optimizedManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(optimizedManager);
+ doReturn(FILE_SYSTEM_OPTIMIZED).when(spy).getBucketLayout(anyString(),
anyString(), any());
+ doNothing().when(spy).generateSnapshotDiffReport(eq(ctx.diffJobKey),
anyString(),
+ eq(ctx.volumeName), eq(ctx.bucketName), eq(ctx.fromSnapshotName),
eq(ctx.toSnapshotName),
+ eq(false), eq(false), eq(true));
+
+ SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
+ ctx.fromSnapshotName, ctx.toSnapshotName, false, false, true);
+ assertTrue(response.getResponse().contains("unordered report"));
+ assertTrue(response.getResponse().contains("ordered report"));
+
+ SnapshotDiffJob updated = spy.getSnapDiffJobTable().get(ctx.diffJobKey);
+ assertNotNull(updated);
+ assertTrue(updated.isOrderedReport());
+ assertEquals(IN_PROGRESS, updated.getStatus());
+ attempt(() -> verify(spy,
atLeast(1)).generateSnapshotDiffReport(eq(ctx.diffJobKey),
+ anyString(), eq(ctx.volumeName), eq(ctx.bucketName),
eq(ctx.fromSnapshotName),
+ eq(ctx.toSnapshotName), eq(false), eq(false), eq(true)),
+ 10, TimeDuration.ONE_SECOND, null, null);
+ }
+
+ @Test
+ public void testSubmitInProgressDoesNotResubmitFsoOrdered()
+ throws IOException {
+ String existingJobId = UUID.randomUUID().toString();
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L, existingJobId,
+ IN_PROGRESS, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 0L, null, 0.0, null);
+ SnapshotDiffManager optimizedManager =
newSnapshotDiffManagerWithOptimizedEnabled(true);
+ optimizedManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(optimizedManager);
+ doReturn(FILE_SYSTEM_OPTIMIZED).when(spy).getBucketLayout(anyString(),
anyString(), any());
+
+ SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
+ ctx.fromSnapshotName, ctx.toSnapshotName, false, false, true);
+ assertThat(response.getResponse())
+ .contains("Status: IN_PROGRESS")
+ .contains("Please get the report using --get-report option")
+ .doesNotContain("unordered report")
+ .doesNotContain("ordered report");
+
+ SnapshotDiffJob updated = spy.getSnapDiffJobTable().get(ctx.diffJobKey);
+ assertNotNull(updated);
+ assertEquals(existingJobId, updated.getJobId());
+ assertFalse(updated.isOrderedReport());
+ assertEquals(IN_PROGRESS, updated.getStatus());
+
+ verify(spy, never()).generateSnapshotDiffReport(anyString(), anyString(),
+ anyString(), anyString(), anyString(), anyString(), anyBoolean(),
anyBoolean(),
+ anyBoolean());
+ }
+
+ @Test
+ public void testSubmitQueuedResubmitsFsoOrdered()
+ throws Exception {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ QUEUED, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 0L, null, 0.0, null);
+ SnapshotDiffManager optimizedManager =
newSnapshotDiffManagerWithOptimizedEnabled(true);
+ optimizedManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(optimizedManager);
+ doReturn(FILE_SYSTEM_OPTIMIZED).when(spy).getBucketLayout(anyString(),
anyString(), any());
+ doNothing().when(spy).generateSnapshotDiffReport(eq(ctx.diffJobKey),
anyString(),
+ eq(ctx.volumeName), eq(ctx.bucketName), eq(ctx.fromSnapshotName),
eq(ctx.toSnapshotName),
+ eq(false), eq(false), eq(true));
+
+ SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
+ ctx.fromSnapshotName, ctx.toSnapshotName, false, false, true);
+ assertTrue(response.getResponse().contains("unordered report"));
+ assertTrue(response.getResponse().contains("ordered report"));
+
+ SnapshotDiffJob updated = spy.getSnapDiffJobTable().get(ctx.diffJobKey);
+ assertNotNull(updated);
+ assertTrue(updated.isOrderedReport());
+ assertEquals(IN_PROGRESS, updated.getStatus());
+ attempt(() -> verify(spy,
atLeast(1)).generateSnapshotDiffReport(eq(ctx.diffJobKey),
+ anyString(), eq(ctx.volumeName), eq(ctx.bucketName),
eq(ctx.fromSnapshotName),
+ eq(ctx.toSnapshotName), eq(false), eq(false), eq(true)),
+ 10, TimeDuration.ONE_SECOND, null, null);
+ }
+
+ @Test
+ public void testSubmitPollOmitsDependencyReason() throws IOException {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ DONE, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 1L, null, 0.0, null);
+ existing.setReason("Report is dependency ordered.");
+ snapshotDiffManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(snapshotDiffManager);
+ SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
+ ctx.fromSnapshotName, ctx.toSnapshotName, false, false, false);
+
+ assertThat(response.getResponse())
+ .contains("Status: DONE")
+ .contains("Please get the report using --get-report option")
+ .doesNotContain("Report is dependency")
+ .doesNotContain(", reason:");
+ }
+
+ @Test
+ public void testSubmitDoneNoResubmitOptimizedOff()
+ throws IOException {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ DONE, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 1L, null, 0.0, null);
+ snapshotDiffManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(snapshotDiffManager);
+ doReturn(FILE_SYSTEM_OPTIMIZED).when(spy).getBucketLayout(anyString(),
anyString(), any());
+
+ SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
+ ctx.fromSnapshotName, ctx.toSnapshotName, false, false, true);
+ assertTrue(response.getResponse().contains("Please get the report using
--get-report option"));
+
+ verify(spy, never()).generateSnapshotDiffReport(anyString(), anyString(),
+ anyString(), anyString(), anyString(), anyString(), anyBoolean(),
anyBoolean(),
+ anyBoolean());
+ }
+
+ @Test
+ public void testSubmitDoneNoResubmitObsOrdered()
+ throws IOException {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ DONE, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 1L, null, 0.0, null);
+ snapshotDiffManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(snapshotDiffManager);
+ doReturn(OBJECT_STORE).when(spy).getBucketLayout(anyString(), anyString(),
any());
+
+ spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
ctx.fromSnapshotName,
+ ctx.toSnapshotName, false, false, true);
+
+ verify(spy, never()).generateSnapshotDiffReport(anyString(), anyString(),
+ anyString(), anyString(), anyString(), anyString(), anyBoolean(),
anyBoolean(),
+ anyBoolean());
+ assertEquals(DONE,
spy.getSnapDiffJobTable().get(ctx.diffJobKey).getStatus());
+ }
+
+ @Test
+ public void testSubmitDoneNoResubmitAlreadyOrdered()
+ throws IOException {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ DONE, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 1L, null, 0.0, null);
+ existing.setOrderedReport(true);
+ snapshotDiffManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(snapshotDiffManager);
Review Comment:
Same as above: with the default (optimized off) manager this passes without
reaching the `!snapDiffJob.isOrderedReport()` check. With the optimized manager
it also covers the fallback case from the last round (a DONE job whose ordering
fell back keeps `orderedReport=true`, so an `--ordered` submit must not rerun
it). Setting `existing.setReason("Report is dependency unordered.")` here would
make that explicit.
```suggestion
SnapshotDiffManager optimizedManager =
newSnapshotDiffManagerWithOptimizedEnabled(true);
optimizedManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
SnapshotDiffManager spy = spy(optimizedManager);
```
##########
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java:
##########
@@ -1588,11 +1606,190 @@ public void testSubmitSnapshotDiffDoneShortCircuits()
SnapshotDiffManager spy = spy(snapshotDiffManager);
SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName,
- ctx.bucketName, ctx.fromSnapshotName, ctx.toSnapshotName, false,
false);
+ ctx.bucketName, ctx.fromSnapshotName, ctx.toSnapshotName, false,
false, false);
assertNotNull(response);
verify(spy, never()).generateSnapshotDiffReport(anyString(), anyString(),
- anyString(), anyString(), anyString(), anyString(), anyBoolean(),
anyBoolean());
+ anyString(), anyString(), anyString(), anyString(), anyBoolean(),
anyBoolean(),
+ anyBoolean());
+ }
+
+ @Test
+ public void testSubmitDoneResubmitsFsoOrdered()
+ throws Exception {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ DONE, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 1L, null, 0.0, null);
+ SnapshotDiffManager optimizedManager =
newSnapshotDiffManagerWithOptimizedEnabled(true);
+ optimizedManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(optimizedManager);
+ doReturn(FILE_SYSTEM_OPTIMIZED).when(spy).getBucketLayout(anyString(),
anyString(), any());
+ doNothing().when(spy).generateSnapshotDiffReport(eq(ctx.diffJobKey),
anyString(),
+ eq(ctx.volumeName), eq(ctx.bucketName), eq(ctx.fromSnapshotName),
eq(ctx.toSnapshotName),
+ eq(false), eq(false), eq(true));
+
+ SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
+ ctx.fromSnapshotName, ctx.toSnapshotName, false, false, true);
+ assertTrue(response.getResponse().contains("unordered report"));
+ assertTrue(response.getResponse().contains("ordered report"));
+
+ SnapshotDiffJob updated = spy.getSnapDiffJobTable().get(ctx.diffJobKey);
+ assertNotNull(updated);
+ assertTrue(updated.isOrderedReport());
+ assertEquals(IN_PROGRESS, updated.getStatus());
+ attempt(() -> verify(spy,
atLeast(1)).generateSnapshotDiffReport(eq(ctx.diffJobKey),
+ anyString(), eq(ctx.volumeName), eq(ctx.bucketName),
eq(ctx.fromSnapshotName),
+ eq(ctx.toSnapshotName), eq(false), eq(false), eq(true)),
+ 10, TimeDuration.ONE_SECOND, null, null);
+ }
+
+ @Test
+ public void testSubmitInProgressDoesNotResubmitFsoOrdered()
+ throws IOException {
+ String existingJobId = UUID.randomUUID().toString();
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L, existingJobId,
+ IN_PROGRESS, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 0L, null, 0.0, null);
+ SnapshotDiffManager optimizedManager =
newSnapshotDiffManagerWithOptimizedEnabled(true);
+ optimizedManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(optimizedManager);
+ doReturn(FILE_SYSTEM_OPTIMIZED).when(spy).getBucketLayout(anyString(),
anyString(), any());
+
+ SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
+ ctx.fromSnapshotName, ctx.toSnapshotName, false, false, true);
+ assertThat(response.getResponse())
+ .contains("Status: IN_PROGRESS")
+ .contains("Please get the report using --get-report option")
+ .doesNotContain("unordered report")
+ .doesNotContain("ordered report");
+
+ SnapshotDiffJob updated = spy.getSnapDiffJobTable().get(ctx.diffJobKey);
+ assertNotNull(updated);
+ assertEquals(existingJobId, updated.getJobId());
+ assertFalse(updated.isOrderedReport());
+ assertEquals(IN_PROGRESS, updated.getStatus());
+
+ verify(spy, never()).generateSnapshotDiffReport(anyString(), anyString(),
+ anyString(), anyString(), anyString(), anyString(), anyBoolean(),
anyBoolean(),
+ anyBoolean());
+ }
+
+ @Test
+ public void testSubmitQueuedResubmitsFsoOrdered()
+ throws Exception {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ QUEUED, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 0L, null, 0.0, null);
+ SnapshotDiffManager optimizedManager =
newSnapshotDiffManagerWithOptimizedEnabled(true);
+ optimizedManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(optimizedManager);
+ doReturn(FILE_SYSTEM_OPTIMIZED).when(spy).getBucketLayout(anyString(),
anyString(), any());
+ doNothing().when(spy).generateSnapshotDiffReport(eq(ctx.diffJobKey),
anyString(),
+ eq(ctx.volumeName), eq(ctx.bucketName), eq(ctx.fromSnapshotName),
eq(ctx.toSnapshotName),
+ eq(false), eq(false), eq(true));
+
+ SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
+ ctx.fromSnapshotName, ctx.toSnapshotName, false, false, true);
+ assertTrue(response.getResponse().contains("unordered report"));
+ assertTrue(response.getResponse().contains("ordered report"));
+
+ SnapshotDiffJob updated = spy.getSnapDiffJobTable().get(ctx.diffJobKey);
+ assertNotNull(updated);
+ assertTrue(updated.isOrderedReport());
+ assertEquals(IN_PROGRESS, updated.getStatus());
+ attempt(() -> verify(spy,
atLeast(1)).generateSnapshotDiffReport(eq(ctx.diffJobKey),
+ anyString(), eq(ctx.volumeName), eq(ctx.bucketName),
eq(ctx.fromSnapshotName),
+ eq(ctx.toSnapshotName), eq(false), eq(false), eq(true)),
+ 10, TimeDuration.ONE_SECOND, null, null);
+ }
+
+ @Test
+ public void testSubmitPollOmitsDependencyReason() throws IOException {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ DONE, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 1L, null, 0.0, null);
+ existing.setReason("Report is dependency ordered.");
+ snapshotDiffManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(snapshotDiffManager);
+ SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
+ ctx.fromSnapshotName, ctx.toSnapshotName, false, false, false);
+
+ assertThat(response.getResponse())
+ .contains("Status: DONE")
+ .contains("Please get the report using --get-report option")
+ .doesNotContain("Report is dependency")
+ .doesNotContain(", reason:");
+ }
+
+ @Test
+ public void testSubmitDoneNoResubmitOptimizedOff()
+ throws IOException {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ DONE, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 1L, null, 0.0, null);
+ snapshotDiffManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(snapshotDiffManager);
+ doReturn(FILE_SYSTEM_OPTIMIZED).when(spy).getBucketLayout(anyString(),
anyString(), any());
+
+ SubmitSnapshotDiffResponse response =
spy.submitSnapshotDiff(ctx.volumeName, ctx.bucketName,
+ ctx.fromSnapshotName, ctx.toSnapshotName, false, false, true);
+ assertTrue(response.getResponse().contains("Please get the report using
--get-report option"));
+
+ verify(spy, never()).generateSnapshotDiffReport(anyString(), anyString(),
+ anyString(), anyString(), anyString(), anyString(), anyBoolean(),
anyBoolean(),
+ anyBoolean());
+ }
+
+ @Test
+ public void testSubmitDoneNoResubmitObsOrdered()
+ throws IOException {
+ SnapDiffTestContext ctx = setupRandomSnapDiffTestContext();
+ SnapshotDiffJob existing = new SnapshotDiffJob(0L,
UUID.randomUUID().toString(),
+ DONE, ctx.volumeName, ctx.bucketName, ctx.fromSnapshotName,
ctx.toSnapshotName,
+ false, false, 1L, null, 0.0, null);
+ snapshotDiffManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
+
+ SnapshotDiffManager spy = spy(snapshotDiffManager);
Review Comment:
This uses the default `snapshotDiffManager`, which has
`ozone.om.snapshot.diff.optimized.enabled=false`, so `effectiveOrderedReport`
is false and the resubmit check short circuits before it looks at the bucket
layout. The test passes whether or not the OBS check works. Using the optimized
manager like `testSubmitQueuedResubmitsFsoOrdered` does makes it exercise the
OBS branch:
```suggestion
SnapshotDiffManager optimizedManager =
newSnapshotDiffManagerWithOptimizedEnabled(true);
optimizedManager.getSnapDiffJobTable().put(ctx.diffJobKey, existing);
SnapshotDiffManager spy = spy(optimizedManager);
```
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOptimizedSnapshotDiff.java:
##########
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.hadoop.hdds.utils.NativeConstants.ROCKS_TOOLS_NATIVE_PROPERTY;
+import static
org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
+import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
+import static org.apache.ozone.test.OzoneTestBase.uniqueObjectName;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
+import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
+import org.apache.hadoop.ozone.DataTestUtil;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone;
+import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse;
+import org.apache.ozone.test.GenericTestUtils;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
+
+/**
+ * Optimized snapshot diff integration tests (single cluster, native SST diff
enabled).
+ */
+@EnabledIfSystemProperty(named = ROCKS_TOOLS_NATIVE_PROPERTY, matches = "true")
+public class TestOptimizedSnapshotDiff extends OmSnapshotTests {
+
+ public TestOptimizedSnapshotDiff() throws Exception {
+ super(FILE_SYSTEM_OPTIMIZED, false, false, false, false, true);
+ }
+
+ @Override
+ protected boolean isOptimizedSnapshotDiffOnlySuite() {
+ return true;
+ }
+
+ @Test
+ public void testOptimizedDiffUnorderedThenOrdered() throws Exception {
+ String snap1 = "snap1";
+ String snap2 = "snap2";
+ String rootPath = String.format("%s://%s.%s/",
+ OzoneConsts.OZONE_URI_SCHEME, getBucketName(), getVolumeName());
+ try (FileSystem fs = FileSystem.get(new URI(rootPath),
getCluster().getConf())) {
+ createFileKey(fs, "/dir1/dir2/key1");
+ createFileKey(fs, "/dir1/dir3/key1");
+ createSnapshot(getVolumeName(), getBucketName(), snap1);
+ fs.rename(new Path("/dir1/dir2/key1"), new
Path("/dir1/dir3/renamed_key"));
+ fs.delete(new Path("/dir1/dir2"), true);
+ createSnapshot(getVolumeName(), getBucketName(), snap2);
+ }
+
+ List<DiffReportEntry> expectedUnordered = Arrays.asList(
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.DELETE,
"dir1/dir2"),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.RENAME, "dir1/dir2/key1",
"dir1/dir3/renamed_key"));
+
+ List<DiffReportEntry> expectedOrdered = Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.RENAME, "dir1/dir2/key1",
"dir1/dir3/renamed_key"),
+
SnapshotDiffReportOzone.getDiffReportEntry(SnapshotDiffReport.DiffType.DELETE,
"dir1/dir2"));
+
+ SnapshotDiffResponse unordered = waitForDoneReportAfterOrderedSubmit(
+ getVolumeName(), getBucketName(), snap1, snap2, false);
+ assertEquals("Report is dependency unordered.", unordered.getReason());
+ assertEquals(expectedUnordered,
unordered.getSnapshotDiffReport().getDiffList());
+
+ SnapshotDiffResponse ordered = waitForDoneReportAfterOrderedSubmit(
+ getVolumeName(), getBucketName(), snap1, snap2, true);
+ assertEquals("Report is dependency ordered.", ordered.getReason());
+ assertEquals(expectedOrdered,
ordered.getSnapshotDiffReport().getDiffList());
+ }
+
+ @Test
+ public void testOptimizedObsDiff() throws Exception {
+ String volume = uniqueObjectName("vol-opt-obs-");
+ String bucket = uniqueObjectName("buc-opt-obs-");
+ OzoneBucket obsBucket = DataTestUtil.createVolumeAndBucket(
+ getCluster().newClient(), volume, bucket, OBJECT_STORE);
Review Comment:
nit: the client from `getCluster().newClient()` is never closed, so its RPC
connections and threads stay open until the cluster shuts down. Wrapping the
test body in try with resources fixes it:
```java
try (OzoneClient obsClient = getCluster().newClient()) {
OzoneBucket obsBucket = DataTestUtil.createVolumeAndBucket(obsClient,
volume, bucket, OBJECT_STORE);
// rest of the test
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
