hemantk-12 commented on code in PR #4163:
URL: https://github.com/apache/ozone/pull/4163#discussion_r1091320352
##########
hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java:
##########
@@ -1100,4 +1114,108 @@ public void
testPruneOlderSnapshotsWithCompactionHistory(
deleteDirectory(compactionLogDir);
deleteDirectory(sstBackUpDir);
}
+
+ private static Stream<Arguments> sstFilePruningScenarios() {
+ return Stream.of(
+ Arguments.of("Case 1: No compaction.",
+ "",
+ Arrays.asList("000015", "000013", "000011", "000009"),
+ Arrays.asList("000015", "000013", "000011", "000009")
+ ),
+ Arguments.of("Case 2: One level compaction.",
+ "C 000015,000013,000011,000009:000018,000016,000017\n",
+ Arrays.asList("000015", "000013", "000011", "000009", "000018",
+ "000016", "000017", "000026", "000024", "000022", "000020"),
+ Arrays.asList("000015", "000013", "000011", "000009", "000026",
+ "000024", "000022", "000020")
+ ),
+ Arguments.of("Case 3: Multi-level compaction.",
+ "C 000015,000013,000011,000009:000018,000016,000017\n" +
+ "C 000018,000016,000017,000026,000024,000022,000020:000027," +
+ "000030,000028,000031,000029\n" +
+ "C 000027,000030,000028,000031,000029,000039,000037,000035," +
+ "000033:000040,000044,000042,000043,000046,000041,000045\n" +
+ "C 000040,000044,000042,000043,000046,000041,000045,000054," +
+ "000052,000050,000048:000059,000055,000056,000060,000057," +
+ "000058\n",
+ Arrays.asList("000015", "000013", "000011", "000009", "000018",
+ "000016", "000017", "000026", "000024", "000022", "000020",
+ "000027", "000030", "000028", "000031", "000029", "000039",
+ "000037", "000035", "000033", "000040", "000044", "000042",
+ "000043", "000046", "000041", "000045", "000054", "000052",
+ "000050", "000048", "000059", "000055", "000056", "000060",
+ "000057", "000058"),
+ Arrays.asList("000013", "000024", "000035", "000011", "000022",
+ "000033", "000039", "000015", "000026", "000037", "000048",
+ "000009", "000050", "000054", "000020", "000052")
+ )
+ );
+ }
+
+ /**
+ * End-to-end test for SST file pruning.
+ */
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("sstFilePruningScenarios")
+ public void testSstFilePruning(
+ String description,
+ String compactionLog,
+ List<String> initialFiles,
+ List<String> expectedFiles
+ ) throws IOException {
+
+ String sstBackUpDirName = "./test-compaction-sst-backup";
+ File sstBackUpDir = new File(sstBackUpDirName);
+ if (!sstBackUpDir.exists() && !sstBackUpDir.mkdirs()) {
+ fail("Error creating SST backup directory: " + sstBackUpDirName);
+ }
+
+ String compactionLogDirName = "./test-compaction-log";
+ File compactionLogDir = new File(compactionLogDirName);
+ if (!compactionLogDir.exists() && !compactionLogDir.mkdirs()) {
+ fail("Error creating compaction log directory: " + compactionLogDirName);
+ }
+
+ createFileWithContext(compactionLogDirName + "/compaction_log" +
+ COMPACTION_LOG_FILE_NAME_SUFFIX,
+ compactionLog);
+
+ for (String fileName : initialFiles) {
+ createFileWithContext(sstBackUpDir + "/" + fileName + SST_FILE_EXTENSION,
+ fileName);
+ }
+
+ RocksDBCheckpointDiffer differ =
+ new RocksDBCheckpointDiffer(sstBackUpDirName,
+ compactionLogDirName,
+ null,
+ MINUTES.toMillis(10));
+
+ differ.loadAllCompactionLogs();
+ differ.pruneSttFiles();
Review Comment:
```suggestion
differ.pruneSstFiles();
```
##########
hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java:
##########
@@ -1271,6 +1279,30 @@ private SnapshotLogInfo(long snapshotGenerationId,
}
}
+ /**
+ * Defines the task that removes SST files from backup directory which are
+ * not needed to generate snapshot diff using compaction DAG to clean
+ * the disk space.
+ * We can’t simply delete input files in the compaction completed listener
+ * because it is not known which of input files are from previous compaction
+ * and which were created after the compaction.
+ * We can remove SST files which were created from the compaction because
+ * those are not needed to generate snapshot diff. These files are basically
+ * non-leaf nodes of the DAG.
+ */
+ public void pruneSttFiles() {
Review Comment:
```suggestion
public void pruneSstFiles() {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
