This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 864c8a58b8 HDDS-10312. Speed up TestOMDbCheckpointServlet (#7691)
864c8a58b8 is described below
commit 864c8a58b83dbfad21dd89ddad2cab43020b105e
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue Jan 14 16:31:28 2025 +0100
HDDS-10312. Speed up TestOMDbCheckpointServlet (#7691)
---
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 176 ++++++++++-----------
1 file changed, 81 insertions(+), 95 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
index a94f6ea017..253c993f1d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
@@ -50,6 +50,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -57,12 +58,14 @@ import java.util.stream.Stream;
import com.google.common.collect.Sets;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
@@ -92,9 +95,6 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.io.TempDir;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE;
import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.DATA_PREFIX;
@@ -110,7 +110,6 @@ import static
org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.params.provider.Arguments.arguments;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyBoolean;
import static org.mockito.Mockito.anyInt;
@@ -135,6 +134,8 @@ public class TestOMDbCheckpointServlet {
private File tempFile;
private ServletOutputStream servletOutputStream;
private MiniOzoneCluster cluster = null;
+ private OzoneClient client;
+ private OzoneManager om;
private OMMetrics omMetrics = null;
private HttpServletRequest requestMock = null;
private HttpServletResponse responseMock = null;
@@ -144,23 +145,33 @@ public class TestOMDbCheckpointServlet {
private String snapshotDirName2;
private Path compactionDirPath;
private DBCheckpoint dbCheckpoint;
- private String method;
@TempDir
private Path folder;
private static final String FABRICATED_FILE_NAME = "fabricatedFile.sst";
+ private static final AtomicInteger COUNTER = new AtomicInteger();
- /**
- * Create a MiniDFSCluster for testing.
- * <p>
- * Ozone is made active by setting OZONE_ENABLED = true
- *
- * @throws Exception
- */
@BeforeEach
void init() throws Exception {
conf = new OzoneConfiguration();
+ }
+
+ @AfterEach
+ void shutdown() {
+ IOUtils.closeQuietly(client, cluster);
+ }
+
+ private void setupCluster() throws Exception {
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(1)
+ .build();
+ cluster.waitForClusterToBeReady();
+ client = cluster.newClient();
+ om = cluster.getOzoneManager();
+ omMetrics = om.getMetrics();
+ }
- final Path tempPath = folder.resolve("temp.tar");
+ private void setupMocks() throws Exception {
+ final Path tempPath = folder.resolve("temp" + COUNTER.incrementAndGet() +
".tar");
tempFile = tempPath.toFile();
servletOutputStream = new ServletOutputStream() {
@@ -186,30 +197,12 @@ public class TestOMDbCheckpointServlet {
fileOutputStream.write(b);
}
};
- }
-
- /**
- * Shutdown MiniDFSCluster.
- */
- @AfterEach
- public void shutdown() throws InterruptedException {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- private void setupCluster() throws Exception {
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(1)
- .build();
- cluster.waitForClusterToBeReady();
- omMetrics = cluster.getOzoneManager().getMetrics();
omDbCheckpointServletMock =
mock(OMDBCheckpointServlet.class);
BootstrapStateHandler.Lock lock =
- new OMDBCheckpointServlet.Lock(cluster.getOzoneManager());
+ new OMDBCheckpointServlet.Lock(om);
doCallRealMethod().when(omDbCheckpointServletMock).init();
assertNull(
doCallRealMethod().when(omDbCheckpointServletMock).getDbStore());
@@ -225,7 +218,7 @@ public class TestOMDbCheckpointServlet {
.thenReturn(servletContextMock);
when(servletContextMock.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE))
- .thenReturn(cluster.getOzoneManager());
+ .thenReturn(om);
when(requestMock.getParameter(OZONE_DB_CHECKPOINT_REQUEST_FLUSH))
.thenReturn("true");
@@ -244,17 +237,29 @@ public class TestOMDbCheckpointServlet {
anyBoolean());
}
- @ParameterizedTest
- @MethodSource("getHttpMethods")
- public void testEndpoint(String httpMethod) throws Exception {
- this.method = httpMethod;
-
+ @Test
+ void testWithoutACL() throws Exception {
conf.setBoolean(OZONE_ACL_ENABLED, false);
conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
setupCluster();
- final OzoneManager om = cluster.getOzoneManager();
+ testBootstrapLocking();
+
+ testEndpoint("POST");
+ testEndpoint("GET");
+ testDoPostWithInvalidContentType();
+
+ prepSnapshotData();
+
+ testWriteDbDataWithoutOmSnapshot();
+ testWriteDbDataToStream();
+ testWriteDbDataWithToExcludeFileList();
+ }
+
+ private void testEndpoint(String method) throws Exception {
+ setupMocks();
+
doCallRealMethod().when(omDbCheckpointServletMock).initialize(
om.getMetadataManager().getStore(),
om.getMetrics().getDBCheckpointMetrics(),
@@ -270,7 +275,7 @@ public class TestOMDbCheckpointServlet {
toExcludeList.add("sstFile1.sst");
toExcludeList.add("sstFile2.sst");
- setupHttpMethod(toExcludeList);
+ setupHttpMethod(method, toExcludeList);
when(responseMock.getOutputStream()).thenReturn(servletOutputStream);
@@ -278,7 +283,7 @@ public class TestOMDbCheckpointServlet {
long initialCheckpointCount =
omMetrics.getDBCheckpointMetrics().getNumCheckpoints();
- doEndpoint();
+ doEndpoint(method);
assertThat(tempFile.length()).isGreaterThan(0);
assertThat(omMetrics.getDBCheckpointMetrics().getLastCheckpointCreationTimeTaken())
@@ -292,14 +297,8 @@ public class TestOMDbCheckpointServlet {
any(), any(), eq(toExcludeList), any(), any());
}
- @Test
- public void testDoPostWithInvalidContentType() throws Exception {
- conf.setBoolean(OZONE_ACL_ENABLED, false);
- conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
-
- setupCluster();
-
- final OzoneManager om = cluster.getOzoneManager();
+ private void testDoPostWithInvalidContentType() throws Exception {
+ setupMocks();
doCallRealMethod().when(omDbCheckpointServletMock).initialize(
om.getMetadataManager().getStore(),
@@ -322,11 +321,8 @@ public class TestOMDbCheckpointServlet {
verify(responseMock).setStatus(HttpServletResponse.SC_BAD_REQUEST);
}
- @ParameterizedTest
- @MethodSource("getHttpMethods")
- public void testSpnegoEnabled(String httpMethod) throws Exception {
- this.method = httpMethod;
-
+ @Test
+ void testSpnegoEnabled() throws Exception {
conf.setBoolean(OZONE_ACL_ENABLED, true);
conf.set(OZONE_ADMINISTRATORS, "");
conf.set(OZONE_OM_HTTP_AUTH_TYPE, "kerberos");
@@ -334,7 +330,13 @@ public class TestOMDbCheckpointServlet {
setupCluster();
- final OzoneManager om = cluster.getOzoneManager();
+ testSpnegoEnabled("POST");
+ testSpnegoEnabled("GET");
+ }
+
+ private void testSpnegoEnabled(String method) throws Exception {
+ setupMocks();
+
Collection<String> allowedUsers =
new LinkedHashSet<>(om.getOmAdminUsernames());
allowedUsers.add("recon");
@@ -349,9 +351,9 @@ public class TestOMDbCheckpointServlet {
omDbCheckpointServletMock.init();
- setupHttpMethod(new ArrayList<>());
+ setupHttpMethod(method, new ArrayList<>());
- doEndpoint();
+ doEndpoint(method);
// Response status should be set to 403 Forbidden since there was no user
// principal set in the request
@@ -364,7 +366,7 @@ public class TestOMDbCheckpointServlet {
when(userPrincipalMock.getName()).thenReturn("dn/localhost@REALM");
when(requestMock.getUserPrincipal()).thenReturn(userPrincipalMock);
- doEndpoint();
+ doEndpoint(method);
// Verify that the Response status is set to 403 again for DN user.
verify(responseMock, times(2)).setStatus(HttpServletResponse.SC_FORBIDDEN);
@@ -375,22 +377,21 @@ public class TestOMDbCheckpointServlet {
when(requestMock.getUserPrincipal()).thenReturn(userPrincipalMock);
when(responseMock.getOutputStream()).thenReturn(servletOutputStream);
- doEndpoint();
+ doEndpoint(method);
// Recon user should be able to access the servlet and download the
// snapshot
assertThat(tempFile.length()).isGreaterThan(0);
}
- @Test
- public void testWriteDbDataToStream() throws Exception {
- prepSnapshotData();
+ private void testWriteDbDataToStream() throws Exception {
+ setupMocks();
+
// Set http param to include snapshot data.
when(requestMock.getParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA))
.thenReturn("true");
// Create a "spy" dbstore keep track of the checkpoint.
- OzoneManager om = cluster.getOzoneManager();
DBStore dbStore = om.getMetadataManager().getStore();
DBStore spyDbStore = spy(dbStore);
@@ -517,10 +518,9 @@ public class TestOMDbCheckpointServlet {
}
}
- @Test
- public void testWriteDbDataWithoutOmSnapshot()
+ private void testWriteDbDataWithoutOmSnapshot()
throws Exception {
- prepSnapshotData();
+ setupMocks();
doCallRealMethod().when(omDbCheckpointServletMock).initialize(
any(), any(), anyBoolean(), any(), any(), anyBoolean());
@@ -553,10 +553,9 @@ public class TestOMDbCheckpointServlet {
assertEquals(initialCheckpointSet, finalCheckpointSet);
}
- @Test
- public void testWriteDbDataWithToExcludeFileList()
+ private void testWriteDbDataWithToExcludeFileList()
throws Exception {
- prepSnapshotData();
+ setupMocks();
doCallRealMethod().when(omDbCheckpointServletMock).initialize(
any(), any(), anyBoolean(), any(), any(), anyBoolean());
@@ -604,7 +603,7 @@ public class TestOMDbCheckpointServlet {
/**
* Calls endpoint in regards to parametrized HTTP method.
*/
- private void doEndpoint() {
+ private void doEndpoint(String method) {
if (method.equals("POST")) {
omDbCheckpointServletMock.doPost(requestMock, responseMock);
} else {
@@ -612,20 +611,13 @@ public class TestOMDbCheckpointServlet {
}
}
- /**
- * Parametrizes test with HTTP method.
- * @return HTTP method.
- */
- private static Stream<Arguments> getHttpMethods() {
- return Stream.of(arguments("POST"), arguments("GET"));
- }
-
/**
* Setups HTTP method details depending on parametrized HTTP method.
+ *
* @param toExcludeList SST file names to be excluded.
* @throws IOException
*/
- private void setupHttpMethod(List<String> toExcludeList) throws IOException {
+ private void setupHttpMethod(String method, List<String> toExcludeList)
throws IOException {
if (method.equals("POST")) {
setupPostMethod(toExcludeList);
} else {
@@ -684,11 +676,10 @@ public class TestOMDbCheckpointServlet {
}
private void prepSnapshotData() throws Exception {
- setupCluster();
metaDir = OMStorage.getOmDbDir(conf);
OzoneBucket bucket = TestDataUtil
- .createVolumeAndBucket(cluster.newClient());
+ .createVolumeAndBucket(client);
// Create dummy keys for snapshotting.
TestDataUtil.createKey(bucket, UUID.randomUUID().toString(),
@@ -730,17 +721,16 @@ public class TestOMDbCheckpointServlet {
Path currentLink = Paths.get(compactionDirPath.toString(), "CURRENT");
Files.createLink(currentLink, currentFile);
- dbCheckpoint = cluster.getOzoneManager()
- .getMetadataManager().getStore()
+ dbCheckpoint = om.getMetadataManager()
+ .getStore()
.getCheckpoint(true);
}
private String createSnapshot(String vname, String bname)
throws IOException, InterruptedException, TimeoutException {
- final OzoneManager om = cluster.getOzoneManager();
String snapshotName = UUID.randomUUID().toString();
- OzoneManagerProtocol writeClient = cluster.newClient().getObjectStore()
+ OzoneManagerProtocol writeClient = client.getObjectStore()
.getClientProxy().getOzoneManagerClient();
writeClient.createSnapshot(vname, bname, snapshotName);
@@ -839,22 +829,18 @@ public class TestOMDbCheckpointServlet {
assertEquals(file0, file1, "hl filenames are the same");
}
- @Test
- public void testBootstrapLocking() throws Exception {
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
- cluster.waitForClusterToBeReady();
-
+ private void testBootstrapLocking() throws Exception {
// Get the bootstrap state handlers
- KeyManager keyManager = cluster.getOzoneManager().getKeyManager();
+ KeyManager keyManager = om.getKeyManager();
BootstrapStateHandler keyDeletingService =
keyManager.getDeletingService();
BootstrapStateHandler snapshotDeletingService =
keyManager.getSnapshotDeletingService();
BootstrapStateHandler sstFilteringService =
keyManager.getSnapshotSstFilteringService();
- BootstrapStateHandler differ =
- cluster.getOzoneManager().getMetadataManager()
- .getStore().getRocksDBCheckpointDiffer();
+ BootstrapStateHandler differ = om.getMetadataManager()
+ .getStore()
+ .getRocksDBCheckpointDiffer();
ExecutorService executorService = Executors.newCachedThreadPool();
@@ -863,7 +849,7 @@ public class TestOMDbCheckpointServlet {
OMDBCheckpointServlet spyServlet = spy(omDbCheckpointServlet);
ServletContext servletContext = mock(ServletContext.class);
when(servletContext.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE))
- .thenReturn(cluster.getOzoneManager());
+ .thenReturn(om);
doReturn(servletContext).when(spyServlet).getServletContext();
spyServlet.init();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
