This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 80ba150e7f HDDS-8489. Refactor GET request to POST for OM DB snapshot
(#4695)
80ba150e7f is described below
commit 80ba150e7ffd7bbdba0a07ded65f37f347eb878b
Author: Mladjan Gadzic <[email protected]>
AuthorDate: Wed Jun 21 00:56:21 2023 +0200
HDDS-8489. Refactor GET request to POST for OM DB snapshot (#4695)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +
hadoop-hdds/framework/pom.xml | 5 +
.../hadoop/hdds/utils/DBCheckpointServlet.java | 80 +++++-
.../hadoop/ozone/om/helpers/OMNodeDetails.java | 13 +-
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 1 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 +
.../hdds/scm/TestSCMDbCheckpointServlet.java | 199 ++++++++++++---
.../org/apache/hadoop/hdds/scm/package-info.java | 21 ++
.../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 270 ++++++++++++++++-----
.../om/ratis_snapshot/OmRatisSnapshotProvider.java | 70 +++++-
.../TestOmRatisSnapshotProvider.java | 145 +++++++++++
pom.xml | 1 +
12 files changed, 684 insertions(+), 124 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index f45c455f53..97e8169ae4 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -149,6 +149,8 @@ public final class OzoneConsts {
public static final String RANGER_OZONE_SERVICE_VERSION_KEY =
"#RANGEROZONESERVICEVERSION";
+ public static final String MULTIPART_FORM_DATA_BOUNDARY = "---XXX";
+
/**
* Supports Bucket Versioning.
*/
diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml
index 746129aae2..d806637e23 100644
--- a/hadoop-hdds/framework/pom.xml
+++ b/hadoop-hdds/framework/pom.xml
@@ -71,6 +71,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
</dependency>
+ <dependency>
+ <groupId>commons-fileupload</groupId>
+ <artifactId>commons-fileupload</artifactId>
+ <version>${commons-fileupload.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
index 291dd3b27d..7d19fc4568 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java
@@ -33,6 +33,10 @@ import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
+import org.apache.commons.fileupload.FileItemIterator;
+import org.apache.commons.fileupload.FileItemStream;
+import org.apache.commons.fileupload.servlet.ServletFileUpload;
+import org.apache.commons.fileupload.util.Streams;
import org.apache.hadoop.hdds.server.OzoneAdmins;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
import org.apache.hadoop.hdds.utils.db.DBStore;
@@ -53,6 +57,8 @@ import org.slf4j.LoggerFactory;
*/
public class DBCheckpointServlet extends HttpServlet {
+ private static final String FIELD_NAME =
+ OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST + "[]";
private static final Logger LOG =
LoggerFactory.getLogger(DBCheckpointServlet.class);
private static final long serialVersionUID = 1L;
@@ -94,15 +100,13 @@ public class DBCheckpointServlet extends HttpServlet {
}
/**
- * Process a GET request for the DB checkpoint snapshot.
- *
- * @param request The servlet request we are processing
- * @param response The servlet response we are creating
+ * Generates Snapshot checkpoint as tar ball.
+ * @param request the HTTP servlet request
+ * @param response the HTTP servlet response
+ * @param isFormData indicator whether request is form data
*/
- @Override
- public void doGet(HttpServletRequest request, HttpServletResponse response) {
-
- LOG.info("Received request to obtain DB checkpoint snapshot");
+ private void generateSnapshotCheckpoint(HttpServletRequest request,
+ HttpServletResponse response, boolean isFormData) {
if (dbStore == null) {
LOG.error(
"Unable to process metadata snapshot request. DB Store is null");
@@ -151,7 +155,8 @@ public class DBCheckpointServlet extends HttpServlet {
List<String> receivedSstList = new ArrayList<>();
List<String> excludedSstList = new ArrayList<>();
- String[] sstParam = request.getParameterValues(
+ String[] sstParam = isFormData ?
+ parseFormDataParameters(request) : request.getParameterValues(
OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST);
if (sstParam != null) {
receivedSstList.addAll(
@@ -216,6 +221,63 @@ public class DBCheckpointServlet extends HttpServlet {
}
}
+ /**
+ * Parses request form data parameters.
+ * @param request the HTTP servlet request
+ * @return array of parsed sst form data parameters for exclusion
+ */
+ private static String[] parseFormDataParameters(HttpServletRequest request) {
+ ServletFileUpload upload = new ServletFileUpload();
+ List<String> sstParam = new ArrayList<>();
+
+ try {
+ FileItemIterator iter = upload.getItemIterator(request);
+ while (iter.hasNext()) {
+ FileItemStream item = iter.next();
+ if (!item.isFormField() || !FIELD_NAME.equals(item.getFieldName())) {
+ continue;
+ }
+
+ sstParam.add(Streams.asString(item.openStream()));
+ }
+ } catch (Exception e) {
+ LOG.warn("Exception occured during form data parsing {}",
e.getMessage());
+ }
+
+ return sstParam.size() == 0 ? null : sstParam.toArray(new String[0]);
+ }
+
+ /**
+ * Process a GET request for the DB checkpoint snapshot.
+ *
+ * @param request The servlet request we are processing
+ * @param response The servlet response we are creating
+ */
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response) {
+ LOG.info("Received GET request to obtain DB checkpoint snapshot");
+
+ generateSnapshotCheckpoint(request, response, false);
+ }
+
+ /**
+ * Process a POST request for the DB checkpoint snapshot.
+ *
+ * @param request The servlet request we are processing
+ * @param response The servlet response we are creating
+ */
+ @Override
+ public void doPost(HttpServletRequest request, HttpServletResponse response)
{
+ LOG.info("Received POST request to obtain DB checkpoint snapshot");
+
+ if (!ServletFileUpload.isMultipartContent(request)) {
+ response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+
+ generateSnapshotCheckpoint(request, response, true);
+ }
+
/**
* Write checkpoint to the stream.
*
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
index b061c14b1f..e9f30b1e9e 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OMNodeDetails.java
@@ -31,12 +31,10 @@ import java.net.InetSocketAddress;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
-import java.util.List;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_HTTP_ENDPOINT;
-import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_PORT_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_PORT_KEY;
@@ -164,8 +162,8 @@ public final class OMNodeDetails extends NodeDetails {
}
}
- public URL getOMDBCheckpointEndpointUrl(boolean isHttp, boolean flush,
- List<String> sstList) throws IOException {
+ public URL getOMDBCheckpointEndpointUrl(boolean isHttp, boolean flush)
+ throws IOException {
URL url;
try {
URIBuilder urlBuilder = new URIBuilder().
@@ -175,12 +173,7 @@ public final class OMNodeDetails extends NodeDetails {
addParameter(OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA, "true").
addParameter(OZONE_DB_CHECKPOINT_REQUEST_FLUSH,
flush ? "true" : "false");
- if (sstList != null && !sstList.isEmpty()) {
- for (String s: sstList) {
- urlBuilder.addParameter(
- OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST, s);
- }
- }
+
url = urlBuilder.build().toURL();
} catch (URISyntaxException | MalformedURLException e) {
throw new IOException("Could not get OM DB Checkpoint Endpoint Url", e);
diff --git a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
index 58cbc18b31..d02c3bcceb 100644
--- a/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
+++ b/hadoop-ozone/dist/src/main/license/bin/LICENSE.txt
@@ -314,6 +314,7 @@ Apache License 2.0
commons-logging:commons-logging
commons-net:commons-net
commons-validator:commons-validator
+ commons-fileupload:commons-fileupload
info.picocli:picocli
io.dropwizard.metrics:metrics-core
io.grpc:grpc-api
diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt
b/hadoop-ozone/dist/src/main/license/jar-report.txt
index cb13a1106e..d54591b4a9 100644
--- a/hadoop-ozone/dist/src/main/license/jar-report.txt
+++ b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@ -33,6 +33,7 @@ share/ozone/lib/commons-net.jar
share/ozone/lib/commons-pool2.jar
share/ozone/lib/commons-text.jar
share/ozone/lib/commons-validator.jar
+share/ozone/lib/commons-fileupload.jar
share/ozone/lib/curator-client.jar
share/ozone/lib/curator-framework.jar
share/ozone/lib/derby.jar
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java
index 28184a8ff8..4217d347d1 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java
@@ -19,17 +19,23 @@ package org.apache.hadoop.hdds.scm;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
+import javax.servlet.ServletInputStream;
import javax.servlet.ServletOutputStream;
import javax.servlet.WriteListener;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.List;
import java.util.UUID;
+import java.util.stream.Stream;
-import org.apache.commons.compress.compressors.CompressorException;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics;
import org.apache.hadoop.hdds.scm.server.SCMDBCheckpointServlet;
@@ -39,6 +45,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.commons.io.FileUtils;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
+import static org.apache.hadoop.ozone.OzoneConsts.MULTIPART_FORM_DATA_BOUNDARY;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
import org.junit.jupiter.api.AfterEach;
@@ -46,9 +53,16 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
-import org.mockito.Matchers;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
@@ -66,6 +80,11 @@ public class TestSCMDbCheckpointServlet {
private String clusterId;
private String scmId;
private String omId;
+ private HttpServletRequest requestMock;
+ private HttpServletResponse responseMock;
+ private String method;
+ private SCMDBCheckpointServlet scmDbCheckpointServletMock;
+ private ServletContext servletContextMock;
/**
* Create a MiniDFSCluster for testing.
@@ -89,6 +108,34 @@ public class TestSCMDbCheckpointServlet {
cluster.waitForClusterToBeReady();
scm = cluster.getStorageContainerManager();
scmMetrics = StorageContainerManager.getMetrics();
+
+ requestMock = mock(HttpServletRequest.class);
+ when(requestMock.getParameter(OZONE_DB_CHECKPOINT_REQUEST_FLUSH))
+ .thenReturn("true");
+
+ responseMock = mock(HttpServletResponse.class);
+
+ scmDbCheckpointServletMock = mock(SCMDBCheckpointServlet.class);
+ doCallRealMethod().when(scmDbCheckpointServletMock).init();
+ doCallRealMethod().when(scmDbCheckpointServletMock).initialize(
+ scm.getScmMetadataStore().getStore(),
+ scmMetrics.getDBCheckpointMetrics(),
+ false,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ false);
+ doCallRealMethod().when(scmDbCheckpointServletMock)
+ .writeDbDataToStream(any(), any(), any(), any(), any());
+ doCallRealMethod().when(scmDbCheckpointServletMock).doPost(requestMock,
+ responseMock);
+ doCallRealMethod().when(scmDbCheckpointServletMock).doGet(requestMock,
+ responseMock);
+
+ servletContextMock = mock(ServletContext.class);
+ when(scmDbCheckpointServletMock.getServletContext())
+ .thenReturn(servletContextMock);
+ when(servletContextMock.getAttribute(OzoneConsts.SCM_CONTEXT_ATTRIBUTE))
+ .thenReturn(cluster.getStorageContainerManager());
}
/**
@@ -101,43 +148,25 @@ public class TestSCMDbCheckpointServlet {
}
}
- @Test
- public void testDoGet()
- throws ServletException, IOException, CompressorException,
- InterruptedException {
+ @ParameterizedTest
+ @MethodSource("getHttpMethods")
+ public void testEndpoint(String httpMethod)
+ throws ServletException, IOException, InterruptedException {
+ this.method = httpMethod;
File tempFile = null;
try {
- SCMDBCheckpointServlet scmDbCheckpointServletMock =
- mock(SCMDBCheckpointServlet.class);
-
- doCallRealMethod().when(scmDbCheckpointServletMock).init();
- doCallRealMethod().when(scmDbCheckpointServletMock).initialize(
- scm.getScmMetadataStore().getStore(),
- scmMetrics.getDBCheckpointMetrics(),
- false,
- Collections.emptyList(),
- Collections.emptyList(),
- false);
- doCallRealMethod().when(scmDbCheckpointServletMock)
- .writeDbDataToStream(any(), any(), any(), any(), any());
-
- HttpServletRequest requestMock = mock(HttpServletRequest.class);
- HttpServletResponse responseMock = mock(HttpServletResponse.class);
-
- ServletContext servletContextMock = mock(ServletContext.class);
- when(scmDbCheckpointServletMock.getServletContext())
- .thenReturn(servletContextMock);
-
- when(servletContextMock.getAttribute(OzoneConsts.SCM_CONTEXT_ATTRIBUTE))
- .thenReturn(cluster.getStorageContainerManager());
- when(requestMock.getParameter(OZONE_DB_CHECKPOINT_REQUEST_FLUSH))
- .thenReturn("true");
+ List<String> toExcludeList = new ArrayList<>();
+ toExcludeList.add("sstFile1.sst");
+ toExcludeList.add("sstFile2.sst");
+
+ setupHttpMethod(toExcludeList);
+
doNothing().when(responseMock).setContentType("application/x-tgz");
- doNothing().when(responseMock).setHeader(Matchers.anyString(),
- Matchers.anyString());
+ doNothing().when(responseMock).setHeader(Mockito.anyString(),
+ Mockito.anyString());
- tempFile = File.createTempFile("testDoGet_" + System
+ tempFile = File.createTempFile("testEndpoint_" + System
.currentTimeMillis(), ".tar");
FileOutputStream fileOutputStream = new FileOutputStream(tempFile);
@@ -158,14 +187,11 @@ public class TestSCMDbCheckpointServlet {
}
});
- doCallRealMethod().when(scmDbCheckpointServletMock).doGet(requestMock,
- responseMock);
-
scmDbCheckpointServletMock.init();
long initialCheckpointCount =
scmMetrics.getDBCheckpointMetrics().getNumCheckpoints();
- scmDbCheckpointServletMock.doGet(requestMock, responseMock);
+ doEndpoint();
Assertions.assertTrue(tempFile.length() > 0);
Assertions.assertTrue(
@@ -176,9 +202,106 @@ public class TestSCMDbCheckpointServlet {
getLastCheckpointStreamingTimeTaken() > 0);
Assertions.assertTrue(scmMetrics.getDBCheckpointMetrics().
getNumCheckpoints() > initialCheckpointCount);
+
+ Mockito.verify(scmDbCheckpointServletMock).writeDbDataToStream(any(),
+ any(), any(), eq(toExcludeList), any());
} finally {
FileUtils.deleteQuietly(tempFile);
}
}
+
+ @Test
+ public void testDoPostWithInvalidContentType() throws ServletException {
+ when(requestMock.getContentType()).thenReturn("application/json");
+
+ scmDbCheckpointServletMock.init();
+
+ scmDbCheckpointServletMock.doPost(requestMock, responseMock);
+
+ Mockito.verify(responseMock).setStatus(HttpServletResponse.SC_BAD_REQUEST);
+ }
+
+ /**
+ * Calls endpoint in regards to parametrized HTTP method.
+ */
+ private void doEndpoint() {
+ if (method.equals("POST")) {
+ scmDbCheckpointServletMock.doPost(requestMock, responseMock);
+ } else {
+ scmDbCheckpointServletMock.doGet(requestMock, responseMock);
+ }
+ }
+
+ /**
+ * Parametrizes test with HTTP method.
+ * @return HTTP method.
+ */
+ private static Stream<Arguments> getHttpMethods() {
+ return Stream.of(arguments("POST"), arguments("GET"));
+ }
+
+ /**
+ * Setups HTTP method details depending on parametrized HTTP method.
+ * @param toExcludeList SST file names to be excluded.
+ * @throws IOException
+ */
+ private void setupHttpMethod(List<String> toExcludeList) throws IOException {
+ if (method.equals("POST")) {
+ setupPostMethod(toExcludeList);
+ } else {
+ setupGetMethod(toExcludeList);
+ }
+ }
+
+ /**
+ * Setups details for HTTP POST request.
+ * @param toExcludeList SST file names to be excluded.
+ * @throws IOException
+ */
+ private void setupPostMethod(List<String> toExcludeList)
+ throws IOException {
+ when(requestMock.getMethod()).thenReturn("POST");
+ when(requestMock.getContentType()).thenReturn("multipart/form-data; " +
+ "boundary=" + MULTIPART_FORM_DATA_BOUNDARY);
+
+ // Generate form data
+ String crNl = "\r\n";
+ String contentDisposition = "Content-Disposition: form-data; name=\"" +
+ OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST + "[]\"" + crNl + crNl;
+ String boundary = "--" + MULTIPART_FORM_DATA_BOUNDARY;
+ String endBoundary = boundary + "--" + crNl;
+ StringBuilder sb = new StringBuilder();
+ toExcludeList.forEach(sfn -> {
+ sb.append(boundary).append(crNl);
+ sb.append(contentDisposition);
+ sb.append(sfn).append(crNl);
+ });
+ sb.append(endBoundary);
+
+ // Use generated form data as input stream to the HTTP request
+ InputStream input = new ByteArrayInputStream(
+ sb.toString().getBytes(StandardCharsets.UTF_8));
+ ServletInputStream inputStream = Mockito.mock(ServletInputStream.class);
+ when(requestMock.getInputStream()).thenReturn(inputStream);
+ when(inputStream.read(any(byte[].class), anyInt(), anyInt()))
+ .thenAnswer(invocation -> {
+ byte[] buffer = invocation.getArgument(0);
+ int offset = invocation.getArgument(1);
+ int length = invocation.getArgument(2);
+ return input.read(buffer, offset, length);
+ });
+ }
+
+ /**
+ * Setups details for HTTP GET request.
+ * @param toExcludeList SST file names to be excluded.
+ */
+ private void setupGetMethod(List<String> toExcludeList) {
+ when(requestMock.getMethod()).thenReturn("GET");
+ when(requestMock
+ .getParameterValues(OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST))
+ .thenReturn(toExcludeList.toArray(new String[0]));
+ }
+
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/package-info.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/package-info.java
new file mode 100644
index 0000000000..0628cf68d3
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Package info tests.
+ */
+package org.apache.hadoop.hdds.scm;
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
index ade5196304..2a6b35ae63 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java
@@ -19,13 +19,16 @@
package org.apache.hadoop.ozone.om;
import javax.servlet.ServletContext;
+import javax.servlet.ServletInputStream;
import javax.servlet.ServletOutputStream;
import javax.servlet.WriteListener;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
@@ -69,6 +72,7 @@ import static
org.apache.hadoop.hdds.recon.ReconConfig.ConfigStrings.OZONE_RECON
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS_WILDCARD;
+import static org.apache.hadoop.ozone.OzoneConsts.MULTIPART_FORM_DATA_BOUNDARY;
import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.DB_COMPACTION_SST_BACKUP_DIR;
@@ -76,24 +80,32 @@ import static
org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DIR;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIR;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_INCLUDE_SNAPSHOT_DATA;
import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE;
import org.apache.ozone.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.junit.rules.Timeout;
-import org.mockito.Matchers;
+import org.junit.Assert;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.Mockito;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE;
import static
org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.truncateFileName;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath;
+import static org.junit.jupiter.params.provider.Arguments.arguments;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
@@ -106,6 +118,7 @@ import static org.mockito.Mockito.when;
/**
* Class used for testing the OM DB Checkpoint provider servlet.
*/
+@Timeout(240)
public class TestOMDbCheckpointServlet {
private OzoneConfiguration conf;
private File tempFile;
@@ -121,13 +134,10 @@ public class TestOMDbCheckpointServlet {
private String snapshotDirName2;
private Path compactionDirPath;
private DBCheckpoint dbCheckpoint;
+ private String method;
+ private File folder;
private static final String FABRICATED_FILE_NAME = "fabricatedFile.sst";
- @Rule
- public Timeout timeout = Timeout.seconds(240);
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
/**
* Create a MiniDFSCluster for testing.
* <p>
@@ -135,11 +145,12 @@ public class TestOMDbCheckpointServlet {
*
* @throws Exception
*/
- @Before
- public void init() throws Exception {
+ @BeforeEach
+ public void init(@TempDir File tempDir) throws Exception {
+ folder = tempDir;
conf = new OzoneConfiguration();
- tempFile = File.createTempFile("testDoGet_" + System
+ tempFile = File.createTempFile("temp_" + System
.currentTimeMillis(), ".tar");
FileOutputStream fileOutputStream = new FileOutputStream(tempFile);
@@ -164,7 +175,7 @@ public class TestOMDbCheckpointServlet {
/**
* Shutdown MiniDFSCluster.
*/
- @After
+ @AfterEach
public void shutdown() throws InterruptedException {
if (cluster != null) {
cluster.shutdown();
@@ -202,6 +213,8 @@ public class TestOMDbCheckpointServlet {
doCallRealMethod().when(omDbCheckpointServletMock).doGet(requestMock,
responseMock);
+ doCallRealMethod().when(omDbCheckpointServletMock).doPost(requestMock,
+ responseMock);
doCallRealMethod().when(omDbCheckpointServletMock)
.writeDbDataToStream(any(), any(), any(), any(), any());
@@ -210,15 +223,17 @@ public class TestOMDbCheckpointServlet {
.thenReturn(lock);
}
- @Test
- public void testDoGet() throws Exception {
+ @ParameterizedTest
+ @MethodSource("getHttpMethods")
+ public void testEndpoint(String httpMethod) throws Exception {
+ this.method = httpMethod;
+
conf.setBoolean(OZONE_ACL_ENABLED, false);
conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
setupCluster();
final OzoneManager om = cluster.getOzoneManager();
-
doCallRealMethod().when(omDbCheckpointServletMock).initialize(
om.getMetadataManager().getStore(),
om.getMetrics().getDBCheckpointMetrics(),
@@ -228,8 +243,13 @@ public class TestOMDbCheckpointServlet {
om.isSpnegoEnabled());
doNothing().when(responseMock).setContentType("application/x-tar");
- doNothing().when(responseMock).setHeader(Matchers.anyString(),
- Matchers.anyString());
+ doNothing().when(responseMock).setHeader(anyString(), anyString());
+
+ List<String> toExcludeList = new ArrayList<>();
+ toExcludeList.add("sstFile1.sst");
+ toExcludeList.add("sstFile2.sst");
+
+ setupHttpMethod(toExcludeList);
when(responseMock.getOutputStream()).thenReturn(servletOutputStream);
@@ -237,21 +257,57 @@ public class TestOMDbCheckpointServlet {
long initialCheckpointCount =
omMetrics.getDBCheckpointMetrics().getNumCheckpoints();
- omDbCheckpointServletMock.doGet(requestMock, responseMock);
+ doEndpoint();
- Assert.assertTrue(tempFile.length() > 0);
- Assert.assertTrue(
+ Assertions.assertTrue(tempFile.length() > 0);
+ Assertions.assertTrue(
omMetrics.getDBCheckpointMetrics().
getLastCheckpointCreationTimeTaken() > 0);
- Assert.assertTrue(
+ Assertions.assertTrue(
omMetrics.getDBCheckpointMetrics().
getLastCheckpointStreamingTimeTaken() > 0);
- Assert.assertTrue(omMetrics.getDBCheckpointMetrics().
+ Assertions.assertTrue(omMetrics.getDBCheckpointMetrics().
getNumCheckpoints() > initialCheckpointCount);
+
+ Mockito.verify(omDbCheckpointServletMock).writeDbDataToStream(any(),
+ any(), any(), eq(toExcludeList), any());
}
@Test
- public void testSpnegoEnabled() throws Exception {
+ public void testDoPostWithInvalidContentType() throws Exception {
+ conf.setBoolean(OZONE_ACL_ENABLED, false);
+ conf.set(OZONE_ADMINISTRATORS, OZONE_ADMINISTRATORS_WILDCARD);
+
+ setupCluster();
+
+ final OzoneManager om = cluster.getOzoneManager();
+
+ doCallRealMethod().when(omDbCheckpointServletMock).initialize(
+ om.getMetadataManager().getStore(),
+ om.getMetrics().getDBCheckpointMetrics(),
+ om.getAclsEnabled(),
+ om.getOmAdminUsernames(),
+ om.getOmAdminGroups(),
+ om.isSpnegoEnabled());
+
+ when(requestMock.getContentType()).thenReturn("application/json");
+ doNothing().when(responseMock).setContentType("application/x-tar");
+ doNothing().when(responseMock).setHeader(anyString(),
+ anyString());
+
+ when(responseMock.getOutputStream()).thenReturn(servletOutputStream);
+
+ omDbCheckpointServletMock.init();
+ omDbCheckpointServletMock.doPost(requestMock, responseMock);
+
+ Mockito.verify(responseMock).setStatus(HttpServletResponse.SC_BAD_REQUEST);
+ }
+
+ @ParameterizedTest
+ @MethodSource("getHttpMethods")
+ public void testSpnegoEnabled(String httpMethod) throws Exception {
+ this.method = httpMethod;
+
conf.setBoolean(OZONE_ACL_ENABLED, true);
conf.set(OZONE_ADMINISTRATORS, "");
conf.set(OZONE_OM_HTTP_AUTH_TYPE, "kerberos");
@@ -273,7 +329,10 @@ public class TestOMDbCheckpointServlet {
om.isSpnegoEnabled());
omDbCheckpointServletMock.init();
- omDbCheckpointServletMock.doGet(requestMock, responseMock);
+
+ setupHttpMethod(new ArrayList<>());
+
+ doEndpoint();
// Response status should be set to 403 Forbidden since there was no user
// principal set in the request
@@ -286,7 +345,7 @@ public class TestOMDbCheckpointServlet {
when(userPrincipalMock.getName()).thenReturn("dn/localhost@REALM");
when(requestMock.getUserPrincipal()).thenReturn(userPrincipalMock);
- omDbCheckpointServletMock.doGet(requestMock, responseMock);
+ doEndpoint();
// Verify that the Response status is set to 403 again for DN user.
verify(responseMock, times(2)).setStatus(HttpServletResponse.SC_FORBIDDEN);
@@ -297,11 +356,11 @@ public class TestOMDbCheckpointServlet {
when(requestMock.getUserPrincipal()).thenReturn(userPrincipalMock);
when(responseMock.getOutputStream()).thenReturn(servletOutputStream);
- omDbCheckpointServletMock.doGet(requestMock, responseMock);
+ doEndpoint();
// Recon user should be able to access the servlet and download the
// snapshot
- Assert.assertTrue(tempFile.length() > 0);
+ Assertions.assertTrue(tempFile.length() > 0);
}
@Test
@@ -318,16 +377,16 @@ public class TestOMDbCheckpointServlet {
}
// Untar the file into a temp folder to be examined.
- String testDirName = folder.newFolder().getAbsolutePath();
+ String testDirName = folder.getAbsolutePath();
int testDirLength = testDirName.length() + 1;
String newDbDirName = testDirName + OM_KEY_PREFIX + OM_DB_NAME;
int newDbDirLength = newDbDirName.length() + 1;
File newDbDir = new File(newDbDirName);
- Assert.assertTrue(newDbDir.mkdirs());
+ Assertions.assertTrue(newDbDir.mkdirs());
FileUtil.unTar(tempFile, newDbDir);
// Move snapshot dir to correct location.
- Assert.assertTrue(new File(newDbDirName, OM_SNAPSHOT_DIR)
+ Assertions.assertTrue(new File(newDbDirName, OM_SNAPSHOT_DIR)
.renameTo(new File(newDbDir.getParent(), OM_SNAPSHOT_DIR)));
// Confirm the checkpoint directories match, (after remove extras).
@@ -338,10 +397,10 @@ public class TestOMDbCheckpointServlet {
Set<String> finalCheckpointSet = getFiles(finalCheckpointLocation,
newDbDirLength);
- Assert.assertTrue("hardlink file exists in checkpoint dir",
- finalCheckpointSet.contains(OM_HARDLINK_FILE));
+ Assertions.assertTrue(finalCheckpointSet.contains(OM_HARDLINK_FILE),
+ "hardlink file exists in checkpoint dir");
finalCheckpointSet.remove(OM_HARDLINK_FILE);
- Assert.assertEquals(initialCheckpointSet, finalCheckpointSet);
+ Assertions.assertEquals(initialCheckpointSet, finalCheckpointSet);
int metaDirLength = metaDir.toString().length() + 1;
String shortSnapshotLocation =
@@ -360,8 +419,8 @@ public class TestOMDbCheckpointServlet {
OM_HARDLINK_FILE))) {
for (String line : lines.collect(Collectors.toList())) {
- Assert.assertFalse("CURRENT file is not a hard link",
- line.contains("CURRENT"));
+ Assertions.assertFalse(line.contains("CURRENT"),
+ "CURRENT file is not a hard link");
if (line.contains(FABRICATED_FILE_NAME)) {
fabricatedLinkLines.add(line);
} else {
@@ -378,8 +437,8 @@ public class TestOMDbCheckpointServlet {
Set<String> initialFullSet =
getFiles(Paths.get(metaDir.toString(), OM_SNAPSHOT_DIR),
metaDirLength);
- Assert.assertEquals("expected snapshot files not found",
- initialFullSet, finalFullSet);
+ Assertions.assertEquals(initialFullSet, finalFullSet,
+ "expected snapshot files not found");
}
@Test
@@ -398,7 +457,7 @@ public class TestOMDbCheckpointServlet {
}
// Untar the file into a temp folder to be examined.
- String testDirName = folder.newFolder().getAbsolutePath();
+ String testDirName = folder.getAbsolutePath();
int testDirLength = testDirName.length() + 1;
FileUtil.unTar(tempFile, new File(testDirName));
@@ -410,7 +469,7 @@ public class TestOMDbCheckpointServlet {
Set<String> finalCheckpointSet = getFiles(finalCheckpointLocation,
testDirLength);
- Assert.assertEquals(initialCheckpointSet, finalCheckpointSet);
+ Assertions.assertEquals(initialCheckpointSet, finalCheckpointSet);
}
@Test
@@ -424,7 +483,7 @@ public class TestOMDbCheckpointServlet {
new FileOutputStream(dummyFile), StandardCharsets.UTF_8)) {
writer.write("Dummy data.");
}
- Assert.assertTrue(dummyFile.exists());
+ Assertions.assertTrue(dummyFile.exists());
List<String> toExcludeList = new ArrayList<>();
List<String> excludedList = new ArrayList<>();
toExcludeList.add(dummyFile.getName());
@@ -440,7 +499,7 @@ public class TestOMDbCheckpointServlet {
}
// Untar the file into a temp folder to be examined.
- String testDirName = folder.newFolder().getAbsolutePath();
+ String testDirName = folder.getAbsolutePath();
int testDirLength = testDirName.length() + 1;
FileUtil.unTar(tempFile, new File(testDirName));
@@ -453,7 +512,89 @@ public class TestOMDbCheckpointServlet {
testDirLength);
initialCheckpointSet.removeAll(finalCheckpointSet);
- Assert.assertTrue(initialCheckpointSet.contains(dummyFile.getName()));
+ Assertions.assertTrue(initialCheckpointSet.contains(dummyFile.getName()));
+ }
+
+ /**
+ * Calls endpoint in regards to parametrized HTTP method.
+ */
+ private void doEndpoint() {
+ if (method.equals("POST")) {
+ omDbCheckpointServletMock.doPost(requestMock, responseMock);
+ } else {
+ omDbCheckpointServletMock.doGet(requestMock, responseMock);
+ }
+ }
+
+ /**
+ * Parametrizes test with HTTP method.
+ * @return HTTP method.
+ */
+ private static Stream<Arguments> getHttpMethods() {
+ return Stream.of(arguments("POST"), arguments("GET"));
+ }
+
+ /**
+ * Setups HTTP method details depending on parametrized HTTP method.
+ * @param toExcludeList SST file names to be excluded.
+ * @throws IOException
+ */
+ private void setupHttpMethod(List<String> toExcludeList) throws IOException {
+ if (method.equals("POST")) {
+ setupPostMethod(toExcludeList);
+ } else {
+ setupGetMethod(toExcludeList);
+ }
+ }
+
+ /**
+ * Setups details for HTTP POST request.
+ * @param toExcludeList SST file names to be excluded.
+ * @throws IOException
+ */
+ private void setupPostMethod(List<String> toExcludeList)
+ throws IOException {
+ when(requestMock.getMethod()).thenReturn("POST");
+ when(requestMock.getContentType()).thenReturn("multipart/form-data; " +
+ "boundary=" + MULTIPART_FORM_DATA_BOUNDARY);
+
+ // Generate form data
+ String crNl = "\r\n";
+ String contentDisposition = "Content-Disposition: form-data; name=\"" +
+ OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST + "[]\"" + crNl + crNl;
+ String boundary = "--" + MULTIPART_FORM_DATA_BOUNDARY;
+ String endBoundary = boundary + "--" + crNl;
+ StringBuilder sb = new StringBuilder();
+ toExcludeList.forEach(sfn -> {
+ sb.append(boundary).append(crNl);
+ sb.append(contentDisposition);
+ sb.append(sfn).append(crNl);
+ });
+ sb.append(endBoundary);
+
+ // Use generated form data as input stream to the HTTP request
+ InputStream input = new ByteArrayInputStream(
+ sb.toString().getBytes(StandardCharsets.UTF_8));
+ ServletInputStream inputStream = Mockito.mock(ServletInputStream.class);
+ when(requestMock.getInputStream()).thenReturn(inputStream);
+ when(inputStream.read(any(byte[].class), anyInt(), anyInt()))
+ .thenAnswer(invocation -> {
+ byte[] buffer = invocation.getArgument(0);
+ int offset = invocation.getArgument(1);
+ int length = invocation.getArgument(2);
+ return input.read(buffer, offset, length);
+ });
+ }
+
+ /**
+ * Setups details for HTTP GET request.
+ * @param toExcludeList SST file names to be excluded.
+ */
+ private void setupGetMethod(List<String> toExcludeList) {
+ when(requestMock.getMethod()).thenReturn("GET");
+ when(requestMock
+ .getParameterValues(OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST))
+ .thenReturn(toExcludeList.toArray(new String[0]));
}
private void prepSnapshotData() throws Exception {
@@ -478,8 +619,8 @@ public class TestOMDbCheckpointServlet {
Path fabricatedSnapshot = Paths.get(
new File(snapshotDirName).getParent(),
"fabricatedSnapshot");
- Assert.assertTrue(fabricatedSnapshot.toFile().mkdirs());
- Assert.assertTrue(Paths.get(fabricatedSnapshot.toString(),
+ fabricatedSnapshot.toFile().mkdirs();
+ Assertions.assertTrue(Paths.get(fabricatedSnapshot.toString(),
FABRICATED_FILE_NAME).toFile().createNewFile());
// Create fabricated links to snapshot dirs
@@ -567,31 +708,32 @@ public class TestOMDbCheckpointServlet {
String realDir = null;
for (String dir: directories) {
if (Paths.get(testDirName, dir, FABRICATED_FILE_NAME).toFile().exists())
{
- Assert.assertNull(
- "Exactly one copy of the fabricated file exists in the tarball",
- realDir);
+ Assertions.assertNull(realDir,
+ "Exactly one copy of the fabricated file exists in the tarball");
realDir = dir;
}
}
- Assert.assertNotNull("real directory found", realDir);
+ Assertions.assertNotNull(realDir, "real directory found");
directories.remove(realDir);
Iterator<String> directoryIterator = directories.iterator();
String dir0 = directoryIterator.next();
String dir1 = directoryIterator.next();
- Assert.assertNotEquals("link directories are different", dir0, dir1);
+ Assertions.assertNotEquals("link directories are different", dir0, dir1);
for (String line : lines) {
String[] files = line.split("\t");
- Assert.assertTrue("fabricated entry contains valid first directory",
- files[0].startsWith(dir0) || files[0].startsWith(dir1));
- Assert.assertTrue("fabricated entry contains correct real directory",
- files[1].startsWith(realDir));
+ Assertions.assertTrue(
+ files[0].startsWith(dir0) || files[0].startsWith(dir1),
+ "fabricated entry contains valid first directory");
+ Assertions.assertTrue(files[1].startsWith(realDir),
+ "fabricated entry contains correct real directory");
Path path0 = Paths.get(files[0]);
Path path1 = Paths.get(files[1]);
- Assert.assertTrue("fabricated entries contains correct file name",
+ Assertions.assertTrue(
path0.getFileName().toString().equals(FABRICATED_FILE_NAME) &&
- path1.getFileName().toString().equals(FABRICATED_FILE_NAME));
+ path1.getFileName().toString().equals(FABRICATED_FILE_NAME),
+ "fabricated entries contains correct file name");
}
}
@@ -601,13 +743,13 @@ public class TestOMDbCheckpointServlet {
String shortSnapshotLocation2,
String line) {
String[] files = line.split("\t");
- Assert.assertTrue("hl entry starts with valid snapshot dir",
- files[0].startsWith(shortSnapshotLocation) ||
- files[0].startsWith(shortSnapshotLocation2));
+ Assertions.assertTrue(files[0].startsWith(shortSnapshotLocation) ||
+ files[0].startsWith(shortSnapshotLocation2),
+ "hl entry starts with valid snapshot dir");
String file0 = files[0].substring(shortSnapshotLocation.length() + 1);
String file1 = files[1];
- Assert.assertEquals("hl filenames are the same", file0, file1);
+ Assertions.assertEquals(file0, file1, "hl filenames are the same");
}
@Test
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
index b924a2294f..7031ca0f3d 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java
@@ -18,11 +18,13 @@
package org.apache.hadoop.ozone.om.ratis_snapshot;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
@@ -39,7 +41,9 @@ import static java.net.HttpURLConnection.HTTP_CREATED;
import static java.net.HttpURLConnection.HTTP_OK;
import org.apache.commons.io.FileUtils;
+import static org.apache.hadoop.ozone.OzoneConsts.MULTIPART_FORM_DATA_BOUNDARY;
import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
+import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_PROVIDER_CONNECTION_TIMEOUT_KEY;
@@ -78,6 +82,17 @@ public class OmRatisSnapshotProvider extends
RDBSnapshotProvider {
private final boolean spnegoEnabled;
private final URLConnectionFactory connectionFactory;
+ public OmRatisSnapshotProvider(File snapshotDir,
+ Map<String, OMNodeDetails> peerNodesMap, HttpConfig.Policy httpPolicy,
+ boolean spnegoEnabled, URLConnectionFactory connectionFactory) {
+ super(snapshotDir, OM_DB_NAME);
+ this.peerNodesMap = new ConcurrentHashMap<>(peerNodesMap);
+ this.httpPolicy = httpPolicy;
+ this.spnegoEnabled = spnegoEnabled;
+ this.connectionFactory = connectionFactory;
+ }
+
+
public OmRatisSnapshotProvider(MutableConfigurationSource conf,
File omRatisSnapshotDir, Map<String, OMNodeDetails> peerNodeDetails) {
super(omRatisSnapshotDir, OM_DB_NAME);
@@ -127,14 +142,21 @@ public class OmRatisSnapshotProvider extends
RDBSnapshotProvider {
throws IOException {
OMNodeDetails leader = peerNodesMap.get(leaderNodeID);
URL omCheckpointUrl = leader.getOMDBCheckpointEndpointUrl(
- httpPolicy.isHttpEnabled(), true,
- HAUtils.getExistingSstFiles(getCandidateDir()));
+ httpPolicy.isHttpEnabled(), true);
LOG.info("Downloading latest checkpoint from Leader OM {}. Checkpoint " +
"URL: {}", leaderNodeID, omCheckpointUrl);
SecurityUtil.doAsCurrentUser(() -> {
HttpURLConnection connection = (HttpURLConnection)
connectionFactory.openConnection(omCheckpointUrl, spnegoEnabled);
- connection.setRequestMethod("GET");
+
+ connection.setRequestMethod("POST");
+ String contentTypeValue = "multipart/form-data; boundary=" +
+ MULTIPART_FORM_DATA_BOUNDARY;
+ connection.setRequestProperty("Content-Type", contentTypeValue);
+ connection.setDoOutput(true);
+ writeFormData(connection,
+ HAUtils.getExistingSstFiles(getCandidateDir()));
+
connection.connect();
int errorCode = connection.getResponseCode();
if ((errorCode != HTTP_OK) && (errorCode != HTTP_CREATED)) {
@@ -159,10 +181,52 @@ public class OmRatisSnapshotProvider extends
RDBSnapshotProvider {
});
}
+ /**
+ * Writes form data to output stream as any HTTP client would for a
+ * multipart/form-data request.
+ * Proper form data includes separator, content disposition and value
+ * separated by a new line.
+ * Example:
+ * <pre>
+ * -----XXX
+ * Content-Disposition: form-data; name="field1"
+ *
+ * value1</pre>
+ * @param connection HTTP URL connection which output stream is used.
+ * @param sstFiles SST files for exclusion.
+ * @throws IOException if an exception occured during writing to output
+ * stream.
+ */
+ public static void writeFormData(HttpURLConnection connection,
+ List<String> sstFiles) throws IOException {
+ try (DataOutputStream out =
+ new DataOutputStream(connection.getOutputStream())) {
+ String toExcludeSstField =
+ "name=\"" + OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST + "[]" + "\"";
+ String crNl = "\r\n";
+ String contentDisposition =
+ "Content-Disposition: form-data; " + toExcludeSstField + crNl + crNl;
+ String separator = "--" + MULTIPART_FORM_DATA_BOUNDARY;
+
+ if (sstFiles.isEmpty()) {
+ out.writeBytes(separator + crNl);
+ out.writeBytes(contentDisposition);
+ }
+
+ for (String sstFile : sstFiles) {
+ out.writeBytes(separator + crNl);
+ out.writeBytes(contentDisposition);
+ out.writeBytes(sstFile + crNl);
+ }
+ out.writeBytes(separator + "--" + crNl);
+ }
+ }
+
@Override
public void close() throws IOException {
if (connectionFactory != null) {
connectionFactory.destroy();
}
}
+
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis_snapshot/TestOmRatisSnapshotProvider.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis_snapshot/TestOmRatisSnapshotProvider.java
new file mode 100644
index 0000000000..6576399922
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis_snapshot/TestOmRatisSnapshotProvider.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.ratis_snapshot;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hdds.server.http.HttpConfig;
+import org.apache.hadoop.hdfs.web.URLConnectionFactory;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.om.helpers.OMNodeDetails;
+import
org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static java.net.HttpURLConnection.HTTP_OK;
+import static org.apache.hadoop.ozone.OzoneConsts.MULTIPART_FORM_DATA_BOUNDARY;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit tests for {@link OmRatisSnapshotProvider}.
+ */
+public class TestOmRatisSnapshotProvider {
+
+ private OmRatisSnapshotProvider omRatisSnapshotProvider;
+ private URLConnectionFactory connectionFactory;
+ private OMNodeDetails leader;
+ private String leaderNodeId;
+ private static final String CR_NL = "\r\n";
+ public static final String CONTENT_DISPOSITION =
+ "Content-Disposition: form-data; name=\""
+ + OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST + "[]\""
+ + CR_NL + CR_NL;
+ private StringBuilder sb;
+ private File targetFile;
+
+ @BeforeEach
+ public void setup(@TempDir File snapshotDir,
+ @TempDir File downloadDir) throws IOException {
+ targetFile = new File(downloadDir, "newfile");
+
+ Map<String, OMNodeDetails> peerNodesMap = new HashMap<>();
+ leaderNodeId = "1";
+ leader = mock(OMNodeDetails.class);
+ peerNodesMap.put(leaderNodeId, leader);
+
+ HttpConfig.Policy httpPolicy = mock(HttpConfig.Policy.class);
+ connectionFactory = mock(URLConnectionFactory.class);
+
+ omRatisSnapshotProvider =
+ new OmRatisSnapshotProvider(snapshotDir, peerNodesMap, httpPolicy,
+ false, connectionFactory);
+
+ sb = new StringBuilder();
+ sb.append("--" + MULTIPART_FORM_DATA_BOUNDARY + CR_NL);
+ sb.append(CONTENT_DISPOSITION);
+ }
+
+ @Test
+ public void testDownloadSnapshot() throws IOException,
+ AuthenticationException {
+ URL omCheckpointUrl = mock(URL.class);
+ when(leader.getOMDBCheckpointEndpointUrl(anyBoolean(), anyBoolean()))
+ .thenReturn(omCheckpointUrl);
+
+ HttpURLConnection connection = mock(HttpURLConnection.class);
+ when(connectionFactory.openConnection(any(URL.class), anyBoolean()))
+ .thenReturn(connection);
+
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ when(connection.getOutputStream()).thenReturn(outputStream);
+ when(connection.getResponseCode()).thenReturn(HTTP_OK);
+ InputStream inputStream =
+ new ByteArrayInputStream(outputStream.toByteArray());
+ when(connection.getInputStream()).thenReturn(inputStream);
+
+ omRatisSnapshotProvider.downloadSnapshot(leaderNodeId, targetFile);
+
+ sb.append("--" + MULTIPART_FORM_DATA_BOUNDARY + "--" + CR_NL);
+ Assertions.assertEquals(sb.toString(),
+ new String(outputStream.toByteArray(), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ public void testWriteFormDataWithSstFile() throws IOException {
+ HttpURLConnection connection = mock(HttpURLConnection.class);
+ List<String> sstFiles = new ArrayList<>();
+ String fileName = "file1.sst";
+ sstFiles.add(fileName);
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ when(connection.getOutputStream()).thenReturn(outputStream);
+
+
+ OmRatisSnapshotProvider.writeFormData(connection, sstFiles);
+
+ sb.append(fileName).append(CR_NL);
+ sb.append("--" + MULTIPART_FORM_DATA_BOUNDARY + "--" + CR_NL);
+ Assertions.assertEquals(sb.toString(),
+ new String(outputStream.toByteArray(), StandardCharsets.UTF_8));
+ }
+
+ @Test
+ public void testWriteFormDataWithoutSstFile() throws IOException {
+ HttpURLConnection connection = mock(HttpURLConnection.class);
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+ when(connection.getOutputStream()).thenReturn(outputStream);
+
+ OmRatisSnapshotProvider.writeFormData(connection, new ArrayList<>());
+
+ sb.append("--" + MULTIPART_FORM_DATA_BOUNDARY + "--" + CR_NL);
+ Assertions.assertEquals(sb.toString(),
+ new String(outputStream.toByteArray(), StandardCharsets.UTF_8));
+ }
+
+}
diff --git a/pom.xml b/pom.xml
index 0d88152bf7..0d8e0c7153 100644
--- a/pom.xml
+++ b/pom.xml
@@ -128,6 +128,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<commons-pool2.version>2.6.0</commons-pool2.version>
<commons-text.version>1.10.0</commons-text.version>
<commons-validator.version>1.6</commons-validator.version>
+ <commons-fileupload.version>1.5</commons-fileupload.version>
<download-maven-plugin.version>1.6.8</download-maven-plugin.version>
<test.build.dir>${project.build.directory}/test-dir</test.build.dir>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
