swamirishi commented on code in PR #4187:
URL: https://github.com/apache/ozone/pull/4187#discussion_r1248045108
##########
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java:
##########
@@ -987,14 +987,16 @@ public SnapshotDiffResponse snapshotDiff(String
volumeName,
String toSnapshot,
String token,
int pageSize,
- boolean forceFullDiff)
+ boolean forceFullDiff,
+ boolean forceNonNativeDiff)
Review Comment:
done
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java:
##########
@@ -547,6 +572,588 @@ public void testCreateSnapshotMissingMandatoryParams()
throws Exception {
() -> createSnapshot(nullstr, bucket));
}
+ private Set<OmKeyInfo> getDeletedKeysFromRocksDb(
+ OMMetadataManager metadataManager) throws IOException {
+ Set<OmKeyInfo> deletedKeys = Sets.newHashSet();
+ try (TableIterator<String,
+ ? extends Table.KeyValue<String, RepeatedOmKeyInfo>>
+ deletedTableIterator = metadataManager.getDeletedTable()
+ .iterator()) {
+ while (deletedTableIterator.hasNext()) {
+ Table.KeyValue<String, RepeatedOmKeyInfo> val =
+ deletedTableIterator.next();
+ deletedKeys.addAll(val.getValue().getOmKeyInfoList());
+ }
+ }
+
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ deletedDirTableIterator = metadataManager.getDeletedDirTable()
+ .iterator()) {
+ while (deletedDirTableIterator.hasNext()) {
+ deletedKeys.add(deletedDirTableIterator.next().getValue());
+ }
+ }
+ return deletedKeys;
+ }
+
+ private OmKeyInfo getOmKeyInfo(String volume, String bucket,
+ String key) throws IOException {
+ return cluster.getOzoneManager().getKeyManager()
+ .getKeyInfo(new OmKeyArgs.Builder().setVolumeName(volume)
+ .setBucketName(bucket).setKeyName(key).build(), null);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Snapshot snap2 is created
+ * 4) Key k1 is deleted.
+ * 5) Snapdiff b/w snap3 & snap2 taken to assert difference of 1 key
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithLatestUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ bucket.deleteKey(key1);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix =
+ createSnapshot(testVolumeName, testBucketName, snap3);
+ SnapshotDiffReportOzone diff =
+ getSnapDiffReport(testVolumeName, testBucketName, snap1, snap2);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ diff = getSnapDiffReport(testVolumeName, testBucketName, snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Snapshot snap2 is created.
+ * 5) Snapshot snap3 is created.
+ * 6) Snapdiff b/w snap3 & snap1 taken to assert difference of 1 key.
+ * 7) Snapdiff b/w snap3 & snap2 taken to assert difference of 0 key.
+ */
+ @Test
+ public void testSnapDiffHandlingReclaimWithPreviousUse() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName,
testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ }
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is deleted.
+ * 4) Key k1 is recreated.
+ * 5) Snapshot snap2 is created.
+ * 6) Snapdiff b/w Active FS & snap1 taken to assert difference of 2 keys.
+ * 7) Snapdiff b/w Active FS & snap2 taken to assert difference of 0 key.
+ * 8) Checking rocks db to ensure the object created shouldn't be reclaimed
+ * as it is used by snapshot.
+ * 9) Key k1 is deleted.
+ * 10) Snapdiff b/w Active FS & snap1 taken to assert difference of 1 key.
+ * 11) Snapdiff b/w Active FS & snap2 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffReclaimWithKeyRecreation() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ OmKeyInfo keyInfo1 = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ bucket.deleteKey(key1);
+ key1 = createFileKey(bucket, key1);
+ OmKeyInfo keyInfo2 = getOmKeyInfo(testVolumeName, testBucketName, key1);
+ String snap2 = "snap2";
+ createSnapshot(testVolumeName, testBucketName, snap2);
+ String snap3 = "snap3";
+ String activeSnapshotPrefix = createSnapshot(testVolumeName,
testBucketName,
+ snap3);
+ SnapshotDiffReportOzone diff = getSnapDiffReport(testVolumeName,
+ testBucketName, snap1, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 2);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1),
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.CREATE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap3);
+ Assert.assertEquals(diff.getDiffList().size(), 0);
+ bucket.deleteKey(key1);
+ String snap4 = "snap4";
+ activeSnapshotPrefix = createSnapshot(testVolumeName, testBucketName,
+ snap4);
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap1, snap4);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ diff = getSnapDiffReport(testVolumeName, testBucketName,
+ snap2, snap4);
+ Assert.assertEquals(diff.getDiffList().size(), 1);
+ Assert.assertEquals(diff.getDiffList(), Arrays.asList(
+ SnapshotDiffReportOzone.getDiffReportEntry(
+ SnapshotDiffReport.DiffType.DELETE, OZONE_URI_DELIMITER + key1)));
+ }
+
+
+ /**
+ * Testing scenario:
+ * 1) Key k1 is created.
+ * 2) Snapshot snap1 created.
+ * 3) Key k1 is renamed to renamed-k1.
+ * 4) Key renamed-k1 is deleted.
+ * 5) Snapshot snap2 created.
+ * 4) Snapdiff b/w snap2 & snap1 taken to assert difference of 1 key.
+ */
+ @Test
+ public void testSnapDiffReclaimWithKeyRename() throws Exception {
+ String testVolumeName = "vol" + RandomStringUtils.randomNumeric(5);
+ String testBucketName = "bucket1";
+ store.createVolume(testVolumeName);
+ OzoneVolume volume = store.getVolume(testVolumeName);
+ volume.createBucket(testBucketName);
+ OzoneBucket bucket = volume.getBucket(testBucketName);
+ String key1 = "k1";
+ key1 = createFileKeyWithPrefix(bucket, key1);
+ String snap1 = "snap1";
+ createSnapshot(testVolumeName, testBucketName, snap1);
+ String renamedKey = "renamed-" + key1;
+ bucket.renameKey(key1, renamedKey);
+ GenericTestUtils.waitFor(
+ () -> {
+ try {
+ getOmKeyInfo(testVolumeName, testBucketName, renamedKey);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }, 1000, 10000);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
