This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new cd0069113e HDDS-10218. Speed up TestSstFilteringService (#6196)
cd0069113e is described below
commit cd0069113ee5086f8df0981f4c758819dfc7298d
Author: Raju Balpande <[email protected]>
AuthorDate: Mon Feb 12 21:56:57 2024 +0530
HDDS-10218. Speed up TestSstFilteringService (#6196)
---
.../hadoop/ozone/om/TestSstFilteringService.java | 79 +++++++++++++---------
1 file changed, 47 insertions(+), 32 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java
index 8ebf76cbf7..2654f4339a 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java
@@ -36,15 +36,19 @@ import
org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted;
import org.apache.ratis.util.ExitUtils;
-import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.MethodOrderer.OrderAnnotation;
import org.junit.jupiter.api.io.TempDir;
import org.rocksdb.LiveFileMetaData;
import java.io.File;
import java.io.IOException;
+import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -70,24 +74,21 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Test SST Filtering Service.
*/
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@TestMethodOrder(OrderAnnotation.class)
public class TestSstFilteringService {
- public static final String SST_FILE_EXTENSION = ".sst";
- @TempDir
- private File folder;
+ private static final String SST_FILE_EXTENSION = ".sst";
private OzoneManagerProtocol writeClient;
private OzoneManager om;
private OzoneConfiguration conf;
private KeyManager keyManager;
+ private short countTotalSnapshots = 0;
@BeforeAll
- public static void setup() {
+ void setup(@TempDir Path folder) throws Exception {
ExitUtils.disableSystemExit();
- }
-
- @BeforeEach
- void init() throws Exception {
conf = new OzoneConfiguration();
- conf.set(OZONE_METADATA_DIRS, folder.getAbsolutePath());
+ conf.set(OZONE_METADATA_DIRS, folder.toString());
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, 100,
@@ -101,7 +102,7 @@ public class TestSstFilteringService {
om = omTestManagers.getOzoneManager();
}
- @AfterEach
+ @AfterAll
public void cleanup() throws Exception {
if (keyManager != null) {
keyManager.stop();
@@ -132,6 +133,7 @@ public class TestSstFilteringService {
* @throws IOException - on Failure.
*/
@Test
+ @Order(1)
public void testIrrelevantSstFileDeletion()
throws Exception {
RDBStore activeDbStore = (RDBStore) om.getMetadataManager().getStore();
@@ -141,7 +143,8 @@ public class TestSstFilteringService {
final int keyCount = 100;
String volumeName = "vol1";
String bucketName1 = "buck1";
- createVolumeAndBucket(volumeName, bucketName1);
+ createVolume(volumeName);
+ addBucketToVolume(volumeName, bucketName1);
createKeys(volumeName, bucketName1, keyCount / 2);
activeDbStore.getDb().flush(OmMetadataManagerImpl.KEY_TABLE);
@@ -179,13 +182,13 @@ public class TestSstFilteringService {
assertThat(nonLevel0FilesCountAfterCompact).isGreaterThan(0);
String bucketName2 = "buck2";
- createVolumeAndBucket(volumeName, bucketName2);
+ addBucketToVolume(volumeName, bucketName2);
createKeys(volumeName, bucketName2, keyCount);
activeDbStore.getDb().flush(OmMetadataManagerImpl.KEY_TABLE);
List<LiveFileMetaData> allFiles = activeDbStore.getDb().getSstFileList();
String snapshotName1 = "snapshot1";
- writeClient.createSnapshot(volumeName, bucketName2, snapshotName1);
+ createSnapshot(volumeName, bucketName2, snapshotName1);
SnapshotInfo snapshotInfo = om.getMetadataManager().getSnapshotInfoTable()
.get(SnapshotInfo.getTableKey(volumeName, bucketName2, snapshotName1));
assertFalse(snapshotInfo.isSstFiltered());
@@ -222,7 +225,7 @@ public class TestSstFilteringService {
try (BootstrapStateHandler.Lock lock =
filteringService.getBootstrapStateLock().lock()) {
count = filteringService.getSnapshotFilteredCount().get();
- writeClient.createSnapshot(volumeName, bucketName2, snapshotName2);
+ createSnapshot(volumeName, bucketName2, snapshotName2);
assertThrows(TimeoutException.class,
() -> waitForSnapshotsAtLeast(filteringService, count + 1));
@@ -239,14 +242,16 @@ public class TestSstFilteringService {
}
@Test
+ @Order(2)
public void testActiveAndDeletedSnapshotCleanup() throws Exception {
RDBStore activeDbStore = (RDBStore) om.getMetadataManager().getStore();
String volumeName = "volume1";
List<String> bucketNames = Arrays.asList("bucket1", "bucket2");
+ createVolume(volumeName);
// Create 2 Buckets
for (String bucketName : bucketNames) {
- createVolumeAndBucket(volumeName, bucketName);
+ addBucketToVolume(volumeName, bucketName);
}
// Write 25 keys in each bucket, 2 sst files would be generated each for
// keys in a single bucket
@@ -264,8 +269,8 @@ public class TestSstFilteringService {
keyManager.getSnapshotSstFilteringService();
sstFilteringService.pause();
- writeClient.createSnapshot(volumeName, bucketNames.get(0), "snap1");
- writeClient.createSnapshot(volumeName, bucketNames.get(0), "snap2");
+ createSnapshot(volumeName, bucketNames.get(0), "snap1");
+ createSnapshot(volumeName, bucketNames.get(0), "snap2");
SnapshotInfo snapshot1Info = om.getMetadataManager().getSnapshotInfoTable()
.get(SnapshotInfo.getTableKey(volumeName, bucketNames.get(0),
"snap1"));
@@ -283,15 +288,15 @@ public class TestSstFilteringService {
await(10_000, 1_000, () -> snap1Current.exists() && snap2Current.exists());
long snap1SstFileCountBeforeFilter =
Arrays.stream(snapshot1Dir.listFiles())
- .filter(f -> f.getName().endsWith(".sst")).count();
+ .filter(f -> f.getName().endsWith(SST_FILE_EXTENSION)).count();
long snap2SstFileCountBeforeFilter =
Arrays.stream(snapshot2Dir.listFiles())
- .filter(f -> f.getName().endsWith(".sst")).count();
+ .filter(f -> f.getName().endsWith(SST_FILE_EXTENSION)).count();
// delete snap1
writeClient.deleteSnapshot(volumeName, bucketNames.get(0), "snap1");
sstFilteringService.resume();
// Filtering service will only act on snap2 as it is an active snaphot
- waitForSnapshotsAtLeast(sstFilteringService, 2);
+ waitForSnapshotsAtLeast(sstFilteringService, countTotalSnapshots);
long snap1SstFileCountAfterFilter = Arrays.stream(snapshot1Dir.listFiles())
.filter(f -> f.getName().endsWith(SST_FILE_EXTENSION)).count();
long snap2SstFileCountAfterFilter = Arrays.stream(snapshot2Dir.listFiles())
@@ -299,10 +304,12 @@ public class TestSstFilteringService {
// one sst will be filtered in both active but not in deleted snapshot
// as sstFiltering svc won't run on already deleted snapshots but will mark
// it as filtered.
- assertEquals(2, sstFilteringService.getSnapshotFilteredCount().get());
+ assertEquals(countTotalSnapshots,
sstFilteringService.getSnapshotFilteredCount().get());
assertEquals(snap1SstFileCountBeforeFilter, snap1SstFileCountAfterFilter);
- assertEquals(snap2SstFileCountBeforeFilter - 1,
- snap2SstFileCountAfterFilter);
+ // If method with order 1 is run .sst file from /vol1/buck1 and
/vol1/buck2 will be deleted.
+ // As part of this method .sst file from /volume1/bucket2/ will be deleted.
+ // sstFiltering won't run on deleted snapshots in /volume1/bucket1.
+
assertThat(snap2SstFileCountBeforeFilter).isGreaterThan(snap2SstFileCountAfterFilter);
}
private void createKeys(String volumeName,
@@ -315,8 +322,7 @@ public class TestSstFilteringService {
}
}
- private void createVolumeAndBucket(String volumeName,
- String bucketName)
+ private void createVolume(String volumeName)
throws IOException {
OMRequestTestUtils.addVolumeToOM(keyManager.getMetadataManager(),
OmVolumeArgs.newBuilder()
@@ -324,7 +330,10 @@ public class TestSstFilteringService {
.setAdminName("a")
.setVolume(volumeName)
.build());
+ }
+ private void addBucketToVolume(String volumeName, String bucketName)
+ throws IOException {
OMRequestTestUtils.addBucketToOM(keyManager.getMetadataManager(),
OmBucketInfo.newBuilder().setVolumeName(volumeName)
.setBucketName(bucketName)
@@ -365,13 +374,15 @@ public class TestSstFilteringService {
* snapshot bucket.
*/
@Test
+ @Order(3)
public void testSstFilteringService() throws Exception {
RDBStore activeDbStore = (RDBStore) om.getMetadataManager().getStore();
String volumeName = "volume";
List<String> bucketNames = Arrays.asList("bucket", "bucket1", "bucket2");
+ createVolume(volumeName);
for (String bucketName : bucketNames) {
- createVolumeAndBucket(volumeName, bucketName);
+ addBucketToVolume(volumeName, bucketName);
}
int keyCount = 150;
@@ -406,15 +417,14 @@ public class TestSstFilteringService {
List<String> snapshotNames = Arrays.asList("snap", "snap-1", "snap-2");
for (int i = 0; i < 3; i++) {
- writeClient.createSnapshot(volumeName, bucketNames.get(i),
- snapshotNames.get(i));
+ createSnapshot(volumeName, bucketNames.get(i), snapshotNames.get(i));
}
SstFilteringService sstFilteringService =
keyManager.getSnapshotSstFilteringService();
- waitForSnapshotsAtLeast(sstFilteringService, 3);
- assertEquals(3, sstFilteringService.getSnapshotFilteredCount().get());
+ waitForSnapshotsAtLeast(sstFilteringService, countTotalSnapshots);
+ assertEquals(countTotalSnapshots,
sstFilteringService.getSnapshotFilteredCount().get());
Set<String> keyInBucketAfterFilteringRun =
getKeysFromSnapshot(volumeName, bucketNames.get(0),
@@ -469,4 +479,9 @@ public class TestSstFilteringService {
return getKeysFromDb(omSnapshot.getMetadataManager(), volume, bucket);
}
}
+
+ private void createSnapshot(String volumeName, String bucketName, String
snapshotName) throws IOException {
+ writeClient.createSnapshot(volumeName, bucketName, snapshotName);
+ countTotalSnapshots++;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
