DaveTeng0 commented on code in PR #4969:
URL: https://github.com/apache/ozone/pull/4969#discussion_r1246134909
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java:
##########
@@ -1293,4 +1300,144 @@ public void
testCompactionDagDisableForSnapshotMetadata() throws Exception {
assertNull(snapshotDbStore.getRocksDBCheckpointDiffer());
assertEquals(0, snapshotDbStore.getDbOptions().listeners().size());
}
+
+ @Test
+ public void testDayWeekMonthSnapshotCreationExpiration() throws Exception {
+ String volumeA = "vol-a-" + RandomStringUtils.randomNumeric(5);
+ String bucketA = "buc-a-" + RandomStringUtils.randomNumeric(5);
+ store.createVolume(volumeA);
+ OzoneVolume volA = store.getVolume(volumeA);
+ volA.createBucket(bucketA);
+ OzoneBucket volAbucketA = volA.getBucket(bucketA);
+
+ int latestDayIndex = 0;
+ int latestWeekIndex = 0;
+ int latestMonthIndex = 0;
+ int oldestDayIndex = latestDayIndex;
+ int oldestWeekIndex = latestWeekIndex;
+ int oldestMonthIndex = latestMonthIndex;
+ int daySnapshotRetentionPeriodDays = 7;
+ int weekSnapshotRetentionPeriodWeek = 1;
+ int monthSnapshotRetentionPeriodMonth = 1;
+ int[] updatedDayIndexArr;
+ int[] updatedWeekIndexArr;
+ int[] updatedMonthIndexArr;
+ for (int i = 0; i < 2; i++) {
+ for (int j = 0; j < 4; j++) {
+ for (int k = 0; k < 7; k++) {
+ // if there are seven day's snapshots in cluster already,
+ // remove the oldest day snapshot then create the latest day snapshot
+ updatedDayIndexArr = checkSnapshotExpirationThenCreateLatest(
+ SNAPSHOT_DAY_PREFIX, oldestDayIndex, latestDayIndex,
+ daySnapshotRetentionPeriodDays, volumeA, bucketA, volAbucketA);
+ oldestDayIndex = updatedDayIndexArr[0];
+ latestDayIndex = updatedDayIndexArr[1];
+ }
+ // if there is one week's snapshot in cluster already,
+ // remove the oldest week snapshot then create the latest week snapshot
+ updatedWeekIndexArr = checkSnapshotExpirationThenCreateLatest(
+ SNAPSHOT_WEEK_PREFIX, oldestWeekIndex, latestWeekIndex,
+ weekSnapshotRetentionPeriodWeek, volumeA, bucketA, volAbucketA);
+ oldestWeekIndex = updatedWeekIndexArr[0];
+ latestWeekIndex = updatedWeekIndexArr[1];
+ }
+ // if there is one month's snapshot in cluster already,
+ // remove the oldest month snapshot then create the latest month snapshot
+ updatedMonthIndexArr = checkSnapshotExpirationThenCreateLatest(
+ SNAPSHOT_MONTH_PREFIX, oldestMonthIndex, latestMonthIndex,
+ monthSnapshotRetentionPeriodMonth, volumeA, bucketA, volAbucketA);
+ oldestMonthIndex = updatedMonthIndexArr[0];
+ latestMonthIndex = updatedMonthIndexArr[1];
+ }
+
+ checkDayWeekMonthSnapshotData(volAbucketA,
+ latestDayIndex,
+ latestWeekIndex,
+ latestMonthIndex,
+ KEY_TEST_DATA);
+ }
+
+ private int[] checkSnapshotExpirationThenCreateLatest(String snapshotPrefix,
+ int oldestIndex, int latestIndex, int snapshotRetentionPeriod,
+ String volumeNameStr, String bucketNameStr, OzoneBucket
ozoneBucketClient)
+ throws Exception {
+ if (latestIndex - oldestIndex >= snapshotRetentionPeriod) {
+ store.deleteSnapshot(volumeNameStr, bucketNameStr,
+ snapshotPrefix + oldestIndex);
+ oldestIndex++;
+ }
+ createFileKeyWithData(ozoneBucketClient, KEY_PREFIX + latestIndex,
+ KEY_TEST_DATA);
+ createSnapshot(volumeNameStr, bucketNameStr, snapshotPrefix + latestIndex);
+ latestIndex++;
+ return new int[]{oldestIndex, latestIndex};
+ }
+
+ private void checkDayWeekMonthSnapshotData(OzoneBucket ozoneBucketClient,
+ int latestDayIndex,
+ int latestWeekIndex,
+ int latestMonthIndex,
+ String keyTestData)
+ throws Exception {
+ String curKey = "";
+ for (int i = 0; i < latestDayIndex; i++) {
+ curKey = KEY_PREFIX + i;
+ //validate keys metadata
+ OzoneKeyDetails ozoneKeyDetails = ozoneBucketClient.getKey(curKey);
+ Assert.assertEquals(curKey, ozoneKeyDetails.getName());
+ Assert.assertEquals(ozoneBucketClient.getName(),
+ ozoneKeyDetails.getBucketName());
+ Assert.assertEquals(ozoneBucketClient.getVolumeName(),
+ ozoneKeyDetails.getVolumeName());
+
+ //validate keys data
+ try (OzoneInputStream ozoneInputStream =
+ ozoneBucketClient.readKey(curKey)) {
+ byte[] fileContent = new byte[keyTestData.length()];
+ ozoneInputStream.read(fileContent);
+ Assert.assertEquals(keyTestData, new String(fileContent, UTF_8));
+ }
+
+ //validate day snapshot data integrity
+ validateSnapshotDataIntegrity(SNAPSHOT_DAY_PREFIX, latestDayIndex,
curKey,
+ ozoneBucketClient);
+ //validate week snapshot data integrity
+ validateSnapshotDataIntegrity(SNAPSHOT_WEEK_PREFIX, latestWeekIndex,
+ curKey, ozoneBucketClient);
+ //validate month snapshot data integrity
+ validateSnapshotDataIntegrity(SNAPSHOT_MONTH_PREFIX, latestMonthIndex,
+ curKey, ozoneBucketClient);
+ }
+
+ }
+
+ private void validateSnapshotDataIntegrity(String snapshotPrefix, int index,
+ String keyName,
+ OzoneBucket ozoneBucketClient)
+ throws Exception {
+ Iterator<? extends OzoneKey> iterator =
+ ozoneBucketClient.listKeys(
+ OmSnapshotManager.getSnapshotPrefix(snapshotPrefix + (index - 1)) +
+ KEY_PREFIX);
+ boolean findKey = false;
+ while (iterator.hasNext()) {
+ OzoneKey ozoneKey = iterator.next();
+ if (ozoneKey.getName().contains(keyName)) {
+ findKey = true;
+ break;
+ }
+ }
Review Comment:
sure, updated
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
