This is an automated email from the ASF dual-hosted git repository.
pvillard31 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 28c02870529 NIFI-15888 Add size limit to Standard Content Viewer
(#11303)
28c02870529 is described below
commit 28c02870529d4e958fd65149610479fd35afd68c
Author: David Handermann <[email protected]>
AuthorDate: Tue Jun 2 15:37:09 2026 -0500
NIFI-15888 Add size limit to Standard Content Viewer (#11303)
---
.../StandardContentViewerController.java | 156 ++++++++++++++-------
.../StandardContentViewerControllerTest.java | 75 +++++++++-
.../org/apache/nifi/web/DownloadableContent.java | 18 ++-
.../apache/nifi/web/StandardNiFiContentAccess.java | 4 +-
.../nifi/web/controller/ControllerFacade.java | 3 +-
.../nifi/web/dao/impl/StandardConnectionDAO.java | 4 +-
6 files changed, 206 insertions(+), 54 deletions(-)
diff --git
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/main/java/org/apache/nifi/web/controller/StandardContentViewerController.java
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/main/java/org/apache/nifi/web/controller/StandardContentViewerController.java
index 5db1cbc65b0..3e4057979e3 100644
---
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/main/java/org/apache/nifi/web/controller/StandardContentViewerController.java
+++
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/main/java/org/apache/nifi/web/controller/StandardContentViewerController.java
@@ -28,6 +28,7 @@ import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.io.DatumReader;
import org.apache.nifi.authorization.AccessDeniedException;
+import org.apache.nifi.stream.io.LimitingInputStream;
import org.apache.nifi.web.ContentAccess;
import org.apache.nifi.web.ContentRequestContext;
import org.apache.nifi.web.DownloadableContent;
@@ -45,6 +46,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
@@ -52,17 +57,24 @@ public class StandardContentViewerController extends
HttpServlet {
static final String CONTENT_ACCESS_ATTRIBUTE = "nifi-content-access";
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private static final int CONTENT_LENGTH_LIMIT = 10_485_760;
+
+ private static final Map<String, ContentType> CONTENT_TYPES =
Arrays.stream(ContentType.values())
+ .collect(
+ Collectors.toMap(ContentType::name, Function.identity())
+ );
+
private static final Logger logger =
LoggerFactory.getLogger(StandardContentViewerController.class);
@Override
public void doGet(final HttpServletRequest request, final
HttpServletResponse response) throws IOException {
final ContentRequestContext requestContext = new
HttpServletContentRequestContext(request);
- // get the content
final ServletContext servletContext = request.getServletContext();
final ContentAccess contentAccess = (ContentAccess)
servletContext.getAttribute(CONTENT_ACCESS_ATTRIBUTE);
- // get the content
final DownloadableContent downloadableContent;
try {
downloadableContent = contentAccess.getContent(requestContext);
@@ -80,38 +92,61 @@ public class StandardContentViewerController extends
HttpServlet {
return;
}
- response.setStatus(HttpServletResponse.SC_OK);
+ // Set Response Status before committing based on formatted or
unformatted stream handling
+ final int responseStatus = getResponseStatus(downloadableContent);
+ response.setStatus(responseStatus);
+
+ final LimitingInputStream contentStream =
getContentStream(downloadableContent);
final boolean formatted =
Boolean.parseBoolean(request.getParameter("formatted"));
- if (!formatted) {
- final InputStream contentStream = downloadableContent.getContent();
+ if (formatted) {
+ final ContentType formattedContentType =
getFormattedContentType(request, downloadableContent.getType());
+ if (formattedContentType == null) {
+ response.sendError(HttpURLConnection.HTTP_NOT_ACCEPTABLE,
"Unknown Content Type");
+ } else {
+ final String dataUri = requestContext.getDataUri();
+ final long contentLength =
downloadableContent.getContentLength();
+ writeContentFormatted(dataUri, formattedContentType,
contentStream, contentLength, response);
+ }
+ } else {
contentStream.transferTo(response.getOutputStream());
- return;
}
+ }
- // allow the user to drive the data type but fall back to the content
type if necessary
- String displayName = request.getParameter("mimeTypeDisplayName");
- if (displayName == null) {
- final String contentType = downloadableContent.getType();
- displayName = getDisplayName(contentType);
- }
+ private LimitingInputStream getContentStream(final DownloadableContent
downloadableContent) {
+ final InputStream contentStream = downloadableContent.getContent();
+ return new LimitingInputStream(contentStream, CONTENT_LENGTH_LIMIT);
+ }
- if (displayName == null) {
- response.sendError(HttpURLConnection.HTTP_BAD_REQUEST, "Unknown
content type");
- return;
+ private int getResponseStatus(final DownloadableContent
downloadableContent) {
+ final int responseStatus;
+
+ final long contentLength = downloadableContent.getContentLength();
+ if (contentLength > CONTENT_LENGTH_LIMIT) {
+ responseStatus = HttpURLConnection.HTTP_PARTIAL;
+ } else {
+ responseStatus = HttpURLConnection.HTTP_OK;
}
+ return responseStatus;
+ }
+
+ private void writeContentFormatted(
+ final String dataUri,
+ final ContentType contentType,
+ final LimitingInputStream contentStream,
+ final long contentLength,
+ final HttpServletResponse response
+ ) throws IOException {
try {
- switch (displayName) {
- case "json": {
- // format json
- final ObjectMapper mapper = new ObjectMapper();
- final Object objectJson =
mapper.readValue(downloadableContent.getContent(), Object.class);
-
mapper.writerWithDefaultPrettyPrinter().writeValue(response.getOutputStream(),
objectJson);
+ switch (contentType) {
+ case JSON: {
+ final Object objectJson =
OBJECT_MAPPER.readValue(contentStream, Object.class);
+
OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValue(response.getOutputStream(),
objectJson);
break;
}
- case "xml": {
- final StreamSource source = new
StreamSource(downloadableContent.getContent());
+ case XML: {
+ final StreamSource source = new
StreamSource(contentStream);
try (OutputStream outputStream = new
FormattingOutputStream(response.getOutputStream())) {
final StreamResult result = new
StreamResult(outputStream);
final StandardTransformProvider transformProvider =
new StandardTransformProvider();
@@ -121,7 +156,7 @@ public class StandardContentViewerController extends
HttpServlet {
}
break;
}
- case "avro": {
+ case AVRO: {
final StringBuilder sb = new StringBuilder();
sb.append("[");
// Use Avro conversions to display logical type values in
human readable way.
@@ -135,16 +170,12 @@ public class StandardContentViewerController extends
HttpServlet {
genericData.addLogicalTypeConversion(new
TimeConversions.LocalTimestampMicrosConversion());
genericData.addLogicalTypeConversion(new
TimeConversions.LocalTimestampMillisConversion());
final DatumReader<GenericData.Record> datumReader = new
GenericDatumReader<>(null, null, genericData);
- try (final DataFileStream<GenericData.Record>
dataFileReader = new DataFileStream<>(downloadableContent.getContent(),
datumReader)) {
+ try (final DataFileStream<GenericData.Record>
dataFileReader = new DataFileStream<>(contentStream, datumReader)) {
while (dataFileReader.hasNext()) {
final GenericData.Record record =
dataFileReader.next();
final String formattedRecord =
genericData.toString(record);
sb.append(formattedRecord);
sb.append(",");
- // Do not format more than 10 MB of content.
- if (sb.length() > 1024 * 1024 * 2) {
- break;
- }
}
}
@@ -154,16 +185,14 @@ public class StandardContentViewerController extends
HttpServlet {
sb.append("]");
final String json = sb.toString();
- final ObjectMapper mapper = new ObjectMapper();
- final Object objectJson = mapper.readValue(json,
Object.class);
-
-
mapper.writerWithDefaultPrettyPrinter().writeValue(response.getOutputStream(),
objectJson);
+ final Object objectJson = OBJECT_MAPPER.readValue(json,
Object.class);
+
OBJECT_MAPPER.writerWithDefaultPrettyPrinter().writeValue(response.getOutputStream(),
objectJson);
break;
}
- case "yaml": {
+ case YAML: {
Yaml yaml = new Yaml();
// Parse the YAML file
- final Object yamlObject =
yaml.load(downloadableContent.getContent());
+ final Object yamlObject = yaml.load(contentStream);
DumperOptions options = new DumperOptions();
options.setIndent(2);
options.setPrettyFlow(true);
@@ -174,35 +203,66 @@ public class StandardContentViewerController extends
HttpServlet {
output.dump(yamlObject, response.getWriter());
break;
}
- case "csv":
- case "text": {
- final InputStream contentStream =
downloadableContent.getContent();
+ case CSV:
+ case TEXT: {
contentStream.transferTo(response.getOutputStream());
break;
}
default: {
- response.sendError(HttpURLConnection.HTTP_BAD_REQUEST,
"Unsupported content type: " + displayName);
+ response.sendError(HttpURLConnection.HTTP_NOT_ACCEPTABLE,
"Unsupported Content Type: %s".formatted(contentType));
}
}
} catch (final Throwable t) {
- logger.warn("Unable to format FlowFile content", t);
- response.sendError(HttpURLConnection.HTTP_INTERNAL_ERROR, "Unable
to format FlowFile content");
+ final String message;
+
+ if (contentLength > CONTENT_LENGTH_LIMIT) {
+ message = "FlowFile Content-Length exceeds maximum allowed";
+ logger.warn("Requested FlowFile [{}] Content-Length exceeds
maximum allowed [{} bytes]", dataUri, CONTENT_LENGTH_LIMIT, t);
+ } else {
+ message = "FlowFile formatting failed";
+ logger.warn("Requested FlowFile [{}] formatting failed for
Content Type [{}]", dataUri, contentType, t);
+ }
+
+ response.sendError(HttpURLConnection.HTTP_INTERNAL_ERROR, message);
+ }
+ }
+
+ private ContentType getFormattedContentType(final HttpServletRequest
request, final String downloadableContentType) {
+ final ContentType formattedContentType;
+
+ final String mimeTypeDisplayName =
request.getParameter("mimeTypeDisplayName");
+ if (mimeTypeDisplayName == null) {
+ formattedContentType =
getFormattedContentType(downloadableContentType);
+ } else {
+ final String upperCasedContentType =
mimeTypeDisplayName.toUpperCase();
+ formattedContentType = CONTENT_TYPES.get(upperCasedContentType);
}
+
+ return formattedContentType;
}
- private String getDisplayName(final String contentType) {
+ private ContentType getFormattedContentType(final String contentType) {
return switch (contentType) {
- case "application/json" -> "json";
- case "application/xml", "text/xml" -> "xml";
- case "application/avro-binary", "avro/binary",
"application/avro+binary" -> "avro";
+ case "application/json" -> ContentType.JSON;
+ case "application/xml", "text/xml" -> ContentType.XML;
+ case "application/avro-binary", "avro/binary",
"application/avro+binary" -> ContentType.AVRO;
case "text/x-yaml", "text/yaml", "text/yml", "application/x-yaml",
"application/x-yml", "application/yaml",
- "application/yml" -> "yaml";
- case "text/plain" -> "text";
- case "text/csv" -> "csv";
+ "application/yml" -> ContentType.YAML;
+ case "text/plain" -> ContentType.TEXT;
+ case "text/csv" -> ContentType.CSV;
case null, default -> null;
};
}
+ private enum ContentType {
+ AVRO,
+ CSV,
+ JSON,
+ TEXT,
+ XML,
+ YAML
+ }
+
private static class FormattingOutputStream extends FilterOutputStream {
private static final byte LINE_FEED = 10;
diff --git
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/test/java/org/apache/nifi/web/controller/StandardContentViewerControllerTest.java
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/test/java/org/apache/nifi/web/controller/StandardContentViewerControllerTest.java
index 038726a09b7..500b180f92d 100644
---
a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/test/java/org/apache/nifi/web/controller/StandardContentViewerControllerTest.java
+++
b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-content-viewer/src/test/java/org/apache/nifi/web/controller/StandardContentViewerControllerTest.java
@@ -25,6 +25,8 @@ import org.apache.nifi.web.ContentAccess;
import org.apache.nifi.web.DownloadableContent;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -36,8 +38,11 @@ import java.nio.charset.StandardCharsets;
import java.util.UUID;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@ExtendWith(MockitoExtension.class)
@@ -50,6 +55,8 @@ class StandardContentViewerControllerTest {
private static final String MIME_TYPE_DISPLAY_NAME = "mimeTypeDisplayName";
private static final String XML_DISPLAY_NAME = "xml";
+ private static final int CONTENT_LENGTH_LIMIT_EXCEEDED = 10_485_761;
+ private static final String APPLICATION_OCTET_STREAM =
"application/octet-stream";
private static final String FILENAME = "FlowFile";
private static final String TEXT_XML = "text/xml";
private static final String XML_DOCUMENT = "<?xml version=\"1.0\"
encoding=\"UTF-8\"?><document><element/></document>";
@@ -67,12 +74,42 @@ class StandardContentViewerControllerTest {
@Mock
private ContentAccess contentAccess;
+ @Captor
+ private ArgumentCaptor<String> errorMessageCaptor;
+
private final StandardContentViewerController controller = new
StandardContentViewerController();
+ @Test
+ void testDoGetFormattedContentTypeNotSupported() throws IOException {
+ final InputStream contentStream = new
ByteArrayInputStream(XML_DOCUMENT.getBytes(StandardCharsets.UTF_8));
+ final DownloadableContent downloadableContent = new
DownloadableContent(FILENAME, APPLICATION_OCTET_STREAM, contentStream,
XML_DOCUMENT.length());
+ setDownloadableContent(downloadableContent);
+
when(request.getParameter(eq(FORMATTED_PARAMETER))).thenReturn(Boolean.TRUE.toString());
+
when(request.getParameter(eq(CLIENT_ID_PARAMETER))).thenReturn(UUID.randomUUID().toString());
+
+ controller.doGet(request, response);
+
+ verify(response).sendError(eq(HttpServletResponse.SC_NOT_ACCEPTABLE),
anyString());
+ }
+
+ @Test
+ void testDoGetNotFormattedPartialContent() throws IOException {
+ final InputStream contentStream = new
ByteArrayInputStream(XML_DOCUMENT.getBytes(StandardCharsets.UTF_8));
+ final DownloadableContent downloadableContent = new
DownloadableContent(FILENAME, APPLICATION_OCTET_STREAM, contentStream,
CONTENT_LENGTH_LIMIT_EXCEEDED);
+ setDownloadableContent(downloadableContent);
+
+ final MockServletOutputStream mockServletOutputStream = new
MockServletOutputStream();
+ when(response.getOutputStream()).thenReturn(mockServletOutputStream);
+
+ controller.doGet(request, response);
+
+ verify(response).setStatus(eq(HttpServletResponse.SC_PARTIAL_CONTENT));
+ }
+
@Test
void testDoGetNotFormatted() throws IOException {
final InputStream contentStream = new
ByteArrayInputStream(XML_DOCUMENT.getBytes(StandardCharsets.UTF_8));
- final DownloadableContent downloadableContent = new
DownloadableContent(FILENAME, TEXT_XML, contentStream);
+ final DownloadableContent downloadableContent = new
DownloadableContent(FILENAME, TEXT_XML, contentStream, XML_DOCUMENT.length());
setDownloadableContent(downloadableContent);
final MockServletOutputStream mockServletOutputStream = new
MockServletOutputStream();
@@ -110,9 +147,43 @@ class StandardContentViewerControllerTest {
assertEquals(XML_DOCUMENT_FORMATTED, outputString);
}
+ @Test
+ void testDoGetFormattedXmlMalformed() throws IOException {
+ setDownloadableContentXml(FILENAME);
+
+ final MockServletOutputStream mockServletOutputStream = new
MockServletOutputStream();
+ when(response.getOutputStream()).thenReturn(mockServletOutputStream);
+
+ controller.doGet(request, response);
+
+
verify(response).sendError(eq(HttpServletResponse.SC_INTERNAL_SERVER_ERROR),
errorMessageCaptor.capture());
+ final String errorMessage = errorMessageCaptor.getValue();
+ assertTrue(errorMessage.contains("formatting"));
+ }
+
+ @Test
+ void testDoGetFormattedLimited() throws IOException {
+ final byte[] contentLimitExceededByteArray = new
byte[CONTENT_LENGTH_LIMIT_EXCEEDED];
+ final InputStream contentStream = new
ByteArrayInputStream(contentLimitExceededByteArray);
+ final DownloadableContent downloadableContent = new
DownloadableContent(FILENAME, TEXT_XML, contentStream,
CONTENT_LENGTH_LIMIT_EXCEEDED);
+ setDownloadableContent(downloadableContent);
+
+
when(request.getParameter(eq(FORMATTED_PARAMETER))).thenReturn(Boolean.TRUE.toString());
+
when(request.getParameter(eq(CLIENT_ID_PARAMETER))).thenReturn(UUID.randomUUID().toString());
+
+ final MockServletOutputStream mockServletOutputStream = new
MockServletOutputStream();
+ when(response.getOutputStream()).thenReturn(mockServletOutputStream);
+
+ controller.doGet(request, response);
+
+
verify(response).sendError(eq(HttpServletResponse.SC_INTERNAL_SERVER_ERROR),
errorMessageCaptor.capture());
+ final String errorMessage = errorMessageCaptor.getValue();
+ assertTrue(errorMessage.contains("Content-Length"));
+ }
+
private void setDownloadableContentXml(final String contentXml) {
final InputStream contentStream = new
ByteArrayInputStream(contentXml.getBytes(StandardCharsets.UTF_8));
- final DownloadableContent downloadableContent = new
DownloadableContent(FILENAME, TEXT_XML, contentStream);
+ final DownloadableContent downloadableContent = new
DownloadableContent(FILENAME, TEXT_XML, contentStream, XML_DOCUMENT.length());
setDownloadableContent(downloadableContent);
when(request.getParameter(eq(FORMATTED_PARAMETER))).thenReturn(Boolean.TRUE.toString());
diff --git
a/nifi-framework-api/src/main/java/org/apache/nifi/web/DownloadableContent.java
b/nifi-framework-api/src/main/java/org/apache/nifi/web/DownloadableContent.java
index ee3bc4f4655..c1551edd1c8 100644
---
a/nifi-framework-api/src/main/java/org/apache/nifi/web/DownloadableContent.java
+++
b/nifi-framework-api/src/main/java/org/apache/nifi/web/DownloadableContent.java
@@ -26,11 +26,18 @@ public final class DownloadableContent {
private final String filename;
private final String type;
private final InputStream content;
+ private final long contentLength;
- public DownloadableContent(String filename, String type, InputStream
content) {
+ public DownloadableContent(
+ final String filename,
+ final String type,
+ final InputStream content,
+ final long contentLength
+ ) {
this.filename = filename;
this.type = type;
this.content = content;
+ this.contentLength = contentLength;
}
/**
@@ -59,4 +66,13 @@ public final class DownloadableContent {
public InputStream getContent() {
return content;
}
+
+ /**
+ * Get length in bytes of the content stream
+ *
+ * @return Length in bytes of the content stream
+ */
+ public long getContentLength() {
+ return contentLength;
+ }
}
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiContentAccess.java
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiContentAccess.java
index 5f3743bff30..db5a285a9fa 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiContentAccess.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiContentAccess.java
@@ -123,8 +123,10 @@ public class StandardNiFiContentAccess implements
ContentAccess {
// get the content type
final String contentType = getHeader(responseHeaders,
"content-type");
+ final int contentLength = clientResponse.getLength();
+
// create the downloadable content
- return new DownloadableContent(filename, contentType,
nodeResponse.getInputStream());
+ return new DownloadableContent(filename, contentType,
nodeResponse.getInputStream(), contentLength);
} else {
// example URIs:
//
http://localhost:8080/nifi-api/provenance/events/{id}/content/{input|output}
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
index 4dfbbe89e06..9db50aa2173 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java
@@ -1508,7 +1508,8 @@ public class ControllerFacade implements Authorizable {
// get the content
final InputStream content = flowController.getContent(event,
contentDirection, user.getIdentity(), uri);
- return new DownloadableContent(filename, type, content);
+ final long contentLength = event.getFileSize();
+ return new DownloadableContent(filename, type, content,
contentLength);
} catch (final ContentNotFoundException cnfe) {
throw new ResourceNotFoundException("Unable to find the specified
content.");
} catch (final IOException ioe) {
diff --git
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
index 5b4ab5c1420..c16fb00aefc 100644
---
a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
+++
b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardConnectionDAO.java
@@ -703,7 +703,9 @@ public class StandardConnectionDAO extends ComponentDAO
implements ConnectionDAO
// get the content
final InputStream content = flowController.getContent(flowFile,
user.getIdentity(), requestUri);
- return new DownloadableContent(filename, type, content);
+ final long contentLength = flowFile.getSize();
+
+ return new DownloadableContent(filename, type, content,
contentLength);
} catch (final ContentNotFoundException cnfe) {
throw new ResourceNotFoundException("Unable to find the specified
content.");
} catch (final IOException ioe) {
