This is an automated email from the ASF dual-hosted git repository. turcsanyi pushed a commit to branch support/nifi-1.x in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 9438eb8b86bde3506b92468f6bfa33d988531a93 Author: Lehel <[email protected]> AuthorDate: Thu Feb 16 17:28:19 2023 +0100 NIFI-11190: Added provenance events to GetHubSpot and GetShopify This closes #6965. Signed-off-by: Peter Turcsanyi <[email protected]> --- .../java/org/apache/nifi/processors/hubspot/GetHubSpot.java | 1 + .../org/apache/nifi/processors/hubspot/GetHubSpotTest.java | 9 +++++++++ .../java/org/apache/nifi/processors/shopify/GetShopify.java | 3 +++ .../nifi/processors/shopify/rest/ShopifyRestService.java | 4 ++++ .../shopify/{GetShopifyIT.java => GetShopifyTest.java} | 12 ++++++++---- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/main/java/org/apache/nifi/processors/hubspot/GetHubSpot.java b/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/main/java/org/apache/nifi/processors/hubspot/GetHubSpot.java index 53567ff212..d624b58afe 100644 --- a/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/main/java/org/apache/nifi/processors/hubspot/GetHubSpot.java +++ b/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/main/java/org/apache/nifi/processors/hubspot/GetHubSpot.java @@ -239,6 +239,7 @@ public class GetHubSpot extends AbstractProcessor { if (total.get() > 0) { flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/json"); session.transfer(flowFile, REL_SUCCESS); + session.getProvenanceReporter().receive(flowFile, uri.toString()); } else { getLogger().debug("Empty response when requested HubSpot endpoint: [{}]", endpoint); context.yield(); diff --git a/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/test/java/org/apache/nifi/processors/hubspot/GetHubSpotTest.java b/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/test/java/org/apache/nifi/processors/hubspot/GetHubSpotTest.java index fd8ed18fed..55f371acf6 100644 --- a/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/test/java/org/apache/nifi/processors/hubspot/GetHubSpotTest.java +++ b/nifi-nar-bundles/nifi-hubspot-bundle/nifi-hubspot-processors/src/test/java/org/apache/nifi/processors/hubspot/GetHubSpotTest.java @@ -45,6 +45,7 @@ import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.state.Scope; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.provenance.ProvenanceEventRecord; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; @@ -109,6 +110,8 @@ class GetHubSpotTest { flowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/json"); assertEquals(expectedJsonNode, actualJsonNode); + List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents(); + assertEquals(baseUrl.toString(), provenanceEvents.get(0).getTransitUri()); } @Test @@ -122,6 +125,7 @@ class GetHubSpotTest { final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GetHubSpot.REL_SUCCESS); assertTrue(flowFiles.isEmpty()); + assertTrue(runner.getProvenanceEvents().isEmpty()); } @Test @@ -131,6 +135,7 @@ class GetHubSpotTest { server.enqueue(new MockResponse().setBody(response).setResponseCode(429)); assertThrows(AssertionError.class, () -> runner.run(1)); + assertTrue(runner.getProvenanceEvents().isEmpty()); } @Test @@ -173,6 +178,8 @@ class GetHubSpotTest { final String expectedJsonString = root.toString(); assertEquals(OBJECT_MAPPER.readTree(expectedJsonString), OBJECT_MAPPER.readTree(requestBodyString)); + List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents(); + assertEquals(baseUrl.toString(), provenanceEvents.get(0).getTransitUri()); } @Test @@ -220,6 +227,8 @@ class GetHubSpotTest { final String expectedJsonString = root.toString(); assertEquals(OBJECT_MAPPER.readTree(expectedJsonString), OBJECT_MAPPER.readTree(requestBodyString)); + List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents(); + assertEquals(baseUrl.toString(), provenanceEvents.get(0).getTransitUri()); } static class MockGetHubSpot extends GetHubSpot { diff --git a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/GetShopify.java b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/GetShopify.java index 9994ab999a..f0aee1514d 100644 --- a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/GetShopify.java +++ b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/GetShopify.java @@ -327,6 +327,7 @@ public class GetShopify extends AbstractProcessor { if (objectCountHolder.get() > 0) { flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/json"); session.transfer(flowFile, REL_SUCCESS); + session.getProvenanceReporter().receive(flowFile, shopifyRestService.getBaseUriString()); } else { getLogger().debug("Empty response when requested Shopify resource: [{}]", resourceName); session.remove(flowFile); @@ -415,4 +416,6 @@ public class GetShopify extends AbstractProcessor { throw new ProcessException("Clearing state failed", e); } } + + } diff --git a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/rest/ShopifyRestService.java b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/rest/ShopifyRestService.java index 5bc7ad2f88..f9a8fc1b6a 100644 --- a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/rest/ShopifyRestService.java +++ b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/main/java/org/apache/nifi/processors/shopify/rest/ShopifyRestService.java @@ -76,6 +76,10 @@ public class ShopifyRestService { return retrieveResponse(uri); } + public String getBaseUriString() { + return getBaseUri().build().toString(); + } + private HttpResponseEntity retrieveResponse(URI uri) { return webClientServiceProvider.getWebClientService() .get() diff --git a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyIT.java b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyTest.java similarity index 92% rename from nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyIT.java rename to nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyTest.java index 73ed2a916b..a6f526fe82 100644 --- a/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyIT.java +++ b/nifi-nar-bundles/nifi-shopify-bundle/nifi-shopify-processors/src/test/java/org/apache/nifi/processors/shopify/GetShopifyTest.java @@ -24,6 +24,7 @@ import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.processors.shopify.model.IncrementalLoadingParameter; import org.apache.nifi.processors.shopify.model.ResourceType; import org.apache.nifi.processors.shopify.rest.ShopifyRestService; +import org.apache.nifi.provenance.ProvenanceEventRecord; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.MockProcessContext; @@ -39,7 +40,6 @@ import org.junit.jupiter.api.Test; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.time.Instant; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -49,7 +49,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; -class GetShopifyIT { +class GetShopifyTest { private static final String BASE_URL = "/test/shopify"; private static MockWebServer server; @@ -101,6 +101,8 @@ class GetShopifyIT { List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(GetShopify.REL_SUCCESS); flowFiles.get(0).assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "application/json"); + List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents(); + assertEquals(baseUrl.toString(), provenanceEvents.get(0).getTransitUri()); } @Test @@ -124,6 +126,7 @@ class GetShopifyIT { assertThrows(AssertionError.class, () -> runner.run(1)); assertTrue(((MockProcessContext) runner.getProcessContext()).isYieldCalled(), "In case of 429 the processor should be yielded."); + assertTrue(runner.getProvenanceEvents().isEmpty()); } @Test @@ -148,6 +151,7 @@ class GetShopifyIT { assertTrue(((MockProcessContext) runner.getProcessContext()).isYieldCalled(), "In case of 404 the processor should be yielded."); + assertTrue(runner.getProvenanceEvents().isEmpty()); } @Test @@ -157,8 +161,6 @@ class GetShopifyIT { .setBody(getResourceAsString("collection_listings.json")); server.enqueue(mockResponse); - final Instant expectedExecutionTime = Instant.parse("2022-08-16T10:15:30Z"); - final StandardWebClientServiceProvider standardWebClientServiceProvider = new StandardWebClientServiceProvider(); final CustomGetShopify customGetShopify = new CustomGetShopify(); @@ -177,6 +179,8 @@ class GetShopifyIT { List<MockFlowFile> flowFilesForRelationship = runner.getFlowFilesForRelationship(GetShopify.REL_SUCCESS); assertEquals(1, flowFilesForRelationship.size()); + List<ProvenanceEventRecord> provenanceEvents = runner.getProvenanceEvents(); + assertEquals(baseUrl.toString(), provenanceEvents.get(0).getTransitUri()); } static class CustomGetShopify extends GetShopify {
