This is an automated email from the ASF dual-hosted git repository.
rmerriman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 4d420f4 METRON-1993 Stellar REST_GET should handle responses when
content length is less than zero (merrimanr) closes apache/metron#1331
4d420f4 is described below
commit 4d420f4d1704b2da4e725de3fda4c08d97691611
Author: merrimanr <[email protected]>
AuthorDate: Thu Feb 14 10:42:50 2019 -0600
METRON-1993 Stellar REST_GET should handle responses when content length is
less than zero (merrimanr) closes apache/metron#1331
---
.../metron/stellar/dsl/functions/RestConfig.java | 9 +++
.../stellar/dsl/functions/RestFunctions.java | 30 ++++++--
.../stellar/dsl/functions/RestFunctionsTest.java | 85 ++++++++++++++++++++--
3 files changed, 111 insertions(+), 13 deletions(-)
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
index fdb6935..610717e 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestConfig.java
@@ -98,10 +98,15 @@ public class RestConfig extends HashMap<String, Object> {
*/
public final static String POOLING_DEFAULT_MAX_PER_RUOTE =
"pooling.default.max.per.route";
+ /**
+ * Setting this to true will verify the actual body content length equals
the content length header
+ */
+ public final static String VERIFY_CONTENT_LENGTH = "verify.content.length";
public RestConfig() {
put(TIMEOUT, 1000);
put(RESPONSE_CODES_ALLOWED, Collections.singletonList(200));
+ put(VERIFY_CONTENT_LENGTH, false);
}
public String getBasicAuthUser() {
@@ -164,4 +169,8 @@ public class RestConfig extends HashMap<String, Object> {
public Integer getPoolingDefaultMaxPerRoute() {
return (Integer) get(POOLING_DEFAULT_MAX_PER_RUOTE);
}
+
+ public Boolean verifyContentLength() {
+ return (Boolean) get(VERIFY_CONTENT_LENGTH);
+ }
}
diff --git
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
index f07d54e..d6b03ce 100644
---
a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
+++
b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RestFunctions.java
@@ -244,15 +244,13 @@ public class RestFunctions {
scheduledFuture.cancel(true);
}
int statusCode = response.getStatusLine().getStatusCode();
+ LOG.debug("request = {}; response = {}", httpGet, response);
if (restConfig.getResponseCodesAllowed().contains(statusCode)) {
HttpEntity httpEntity = response.getEntity();
- // Parse the reponse if present, return the empty value override if not
- if (httpEntity != null && httpEntity.getContentLength() > 0) {
- String json = EntityUtils.toString(response.getEntity());
- return JSONUtils.INSTANCE.load(json, JSONUtils.MAP_SUPPLIER);
- }
- return restConfig.getEmptyContentOverride();
+ // Parse the response if present, return the empty value override if
not
+ Optional<Object> parsedResponse = parseResponse(restConfig, httpGet,
httpEntity);
+ return parsedResponse.orElseGet(restConfig::getEmptyContentOverride);
} else {
throw new IOException(String.format("Stellar REST request to %s
expected status code to be one of %s but " +
"failed with http status code %d: %s",
@@ -374,6 +372,26 @@ public class RestFunctions {
return httpClientContext;
}
+ protected Optional<Object> parseResponse(RestConfig restConfig, HttpGet
httpGet, HttpEntity httpEntity) throws IOException {
+ Optional<Object> parsedResponse = Optional.empty();
+ if (httpEntity != null) {
+ int actualContentLength = 0;
+ String json = EntityUtils.toString(httpEntity);
+ if (json != null && !json.isEmpty()) {
+ actualContentLength = json.length();
+ parsedResponse = Optional.of(JSONUtils.INSTANCE.load(json,
JSONUtils.MAP_SUPPLIER));
+ }
+ if (restConfig.verifyContentLength() && actualContentLength !=
httpEntity.getContentLength()) {
+ throw new IOException(String.format("Stellar REST request to %s
returned incorrect or missing content length. " +
+ "Content length in the response was %d but the
actual body content length was %d.",
+ httpGet.getURI().toString(),
+ httpEntity.getContentLength(),
+ actualContentLength));
+ }
+ }
+ return parsedResponse;
+ }
+
/**
* Read bytes from a HDFS path.
* @param inPath
diff --git
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
index 70f7d2f..2008a95 100644
---
a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
+++
b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RestFunctionsTest.java
@@ -19,6 +19,7 @@ package org.apache.metron.stellar.dsl.functions;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.FileUtils;
+import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
@@ -40,8 +41,10 @@ import org.mockserver.client.server.MockServerClient;
import org.mockserver.junit.MockServerRule;
import org.mockserver.junit.ProxyRule;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
+import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
@@ -63,8 +66,10 @@ import static
org.apache.metron.stellar.dsl.functions.RestConfig.PROXY_PORT;
import static
org.apache.metron.stellar.dsl.functions.RestConfig.SOCKET_TIMEOUT;
import static
org.apache.metron.stellar.dsl.functions.RestConfig.STELLAR_REST_SETTINGS;
import static org.apache.metron.stellar.dsl.functions.RestConfig.TIMEOUT;
+import static
org.apache.metron.stellar.dsl.functions.RestConfig.VERIFY_CONTENT_LENGTH;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
@@ -72,6 +77,7 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
import static org.mockserver.model.HttpRequest.request;
import static org.mockserver.model.HttpResponse.response;
@@ -269,7 +275,7 @@ public class RestFunctionsTest {
// Test for default timeout
RestConfig restConfig =
restGet.getRestConfig(Collections.singletonList("uri"), new HashMap<>());
- assertEquals(2, restConfig.size());
+ assertEquals(3, restConfig.size());
assertEquals(1000, restConfig.getTimeout().intValue());
assertEquals(Collections.singletonList(200),
restConfig.getResponseCodesAllowed());
assertNull(restConfig.getBasicAuthUser());
@@ -287,7 +293,7 @@ public class RestFunctionsTest {
{
RestConfig restConfig =
restGet.getRestConfig(Collections.singletonList("uri"), globalRestConfig);
- assertEquals(5, restConfig.size());
+ assertEquals(6, restConfig.size());
assertEquals(1000, restConfig.getTimeout().intValue());
assertEquals(Collections.singletonList(200),
restConfig.getResponseCodesAllowed());
assertEquals(2000, restConfig.getSocketTimeout().intValue());
@@ -306,7 +312,7 @@ public class RestFunctionsTest {
{
RestConfig restConfig = restGet.getRestConfig(Arrays.asList("uri",
functionRestConfig), globalRestConfig);
- assertEquals(5, restConfig.size());
+ assertEquals(6, restConfig.size());
assertEquals(Collections.singletonList(200),
restConfig.getResponseCodesAllowed());
assertEquals(100, restConfig.getTimeout().intValue());
assertEquals(1, restConfig.getSocketTimeout().intValue());
@@ -323,7 +329,7 @@ public class RestFunctionsTest {
{
RestConfig restConfig = restGet.getRestConfig(Arrays.asList("uri",
functionRestConfig), globalRestConfig);
- assertEquals(5, restConfig.size());
+ assertEquals(6, restConfig.size());
assertEquals(Collections.singletonList(200),
restConfig.getResponseCodesAllowed());
assertEquals(100, restConfig.getTimeout().intValue());
assertEquals(2000, restConfig.getSocketTimeout().intValue());
@@ -342,7 +348,7 @@ public class RestFunctionsTest {
{
RestConfig restConfig = restGet.getRestConfig(Arrays.asList("uri",
functionRestConfig), globalRestConfig);
- assertEquals(4, restConfig.size());
+ assertEquals(5, restConfig.size());
assertEquals(Collections.singletonList(200),
restConfig.getResponseCodesAllowed());
assertEquals(100, restConfig.getTimeout().intValue());
assertEquals(2000, restConfig.getSocketTimeout().intValue());
@@ -353,7 +359,7 @@ public class RestFunctionsTest {
{
RestConfig restConfig =
restGet.getRestConfig(Collections.singletonList("uri"), globalRestConfig);
- assertEquals(4, restConfig.size());
+ assertEquals(5, restConfig.size());
assertEquals(Collections.singletonList(200),
restConfig.getResponseCodesAllowed());
assertEquals(1000, restConfig.getTimeout().intValue());
assertEquals(2000, restConfig.getSocketTimeout().intValue());
@@ -364,7 +370,7 @@ public class RestFunctionsTest {
{
RestConfig restConfig =
restGet.getRestConfig(Collections.singletonList("uri"), new HashMap<>());
- assertEquals(2, restConfig.size());
+ assertEquals(3, restConfig.size());
assertEquals(Collections.singletonList(200),
restConfig.getResponseCodesAllowed());
assertEquals(1000, restConfig.getTimeout().intValue());
}
@@ -602,4 +608,69 @@ public class RestFunctionsTest {
verifyNoMoreInteractions(httpClient);
}
+ @Test
+ public void restGetShouldParseResponse() throws Exception {
+ RestFunctions.RestGet restGet = new RestFunctions.RestGet();
+ RestConfig restConfig = new RestConfig();
+ HttpGet httpGet = mock(HttpGet.class);
+ HttpEntity httpEntity = mock(HttpEntity.class);
+
+ // return successfully parsed response
+ when(httpEntity.getContent()).thenReturn(new
ByteArrayInputStream("{\"get\":\"success\"}".getBytes()));
+ Optional<Object> actual = restGet.parseResponse(restConfig, httpGet,
httpEntity);
+ assertTrue(actual.isPresent());
+ assertEquals("success", ((Map<String, Object>) actual.get()).get("get"));
+ }
+
+ @Test
+ public void restGetShouldParseResponseOnNullHttpEntity() throws Exception {
+ RestFunctions.RestGet restGet = new RestFunctions.RestGet();
+ RestConfig restConfig = new RestConfig();
+ HttpGet httpGet = mock(HttpGet.class);
+
+ // return empty on null httpEntity
+ assertEquals(Optional.empty(), restGet.parseResponse(restConfig, httpGet,
null));
+ }
+
+ @Test
+ public void restGetShouldParseResponseOnNullContent() throws Exception {
+ RestFunctions.RestGet restGet = new RestFunctions.RestGet();
+ RestConfig restConfig = new RestConfig();
+ HttpGet httpGet = mock(HttpGet.class);
+ HttpEntity httpEntity = mock(HttpEntity.class);
+
+ // return empty on null content
+ when(httpEntity.getContent()).thenReturn(null);
+ assertEquals(Optional.empty(), restGet.parseResponse(restConfig, httpGet,
httpEntity));
+ }
+
+ @Test
+ public void restGetShouldParseResponseOnEmptyInputStream() throws Exception {
+ RestFunctions.RestGet restGet = new RestFunctions.RestGet();
+ RestConfig restConfig = new RestConfig();
+ HttpGet httpGet = mock(HttpGet.class);
+ HttpEntity httpEntity = mock(HttpEntity.class);
+
+ // return empty on empty input stream
+ when(httpEntity.getContent()).thenReturn(new
ByteArrayInputStream("".getBytes()));
+ assertEquals(Optional.empty(), restGet.parseResponse(restConfig, httpGet,
httpEntity));
+ }
+
+ @Test
+ public void restGetShouldThrowExceptionOnContentLengthMismatch() throws
Exception {
+ thrown.expect(IOException.class);
+ thrown.expectMessage("Stellar REST request to uri returned incorrect or
missing content length. Content length in the response was -1 but the actual
body content length was 17.");
+
+ RestFunctions.RestGet restGet = new RestFunctions.RestGet();
+ RestConfig restConfig = new RestConfig();
+ HttpGet httpGet = mock(HttpGet.class);
+ HttpEntity httpEntity = mock(HttpEntity.class);
+
+ restConfig.put(VERIFY_CONTENT_LENGTH, true);
+ when(httpGet.getURI()).thenReturn(new URI("uri"));
+ when(httpEntity.getContent()).thenReturn(new
ByteArrayInputStream("{\"get\":\"success\"}".getBytes()));
+ when(httpEntity.getContentLength()).thenReturn(-1L);
+ restGet.parseResponse(restConfig, httpGet, httpEntity);
+ }
+
}
