This is an automated email from the ASF dual-hosted git repository. joewitt pushed a commit to branch support/nifi-1.16 in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 04a309af1763747ab9e832d7225df4c3fee0c0bd Author: simonbence <[email protected]> AuthorDate: Wed Apr 20 16:29:38 2022 +0200 NIFI-9776 Adding the possibility to export flow definition with referenced services (#5859) * NIFI-9776 Adding the possibility to export flow definition with referenced services * NIFI-9776 Refining naming based on code review --- nifi-docs/src/main/asciidoc/user-guide.adoc | 6 +- .../org/apache/nifi/web/NiFiServiceFacade.java | 10 ++++ .../apache/nifi/web/StandardNiFiServiceFacade.java | 51 ++++++++++++++++- .../apache/nifi/web/api/ProcessGroupResource.java | 17 +++++- .../nifi/web/StandardNiFiServiceFacadeTest.java | 64 ++++++++++++++++++++++ .../nifi/web/api/TestProcessGroupResource.java | 2 +- .../src/main/webapp/js/nf/canvas/nf-actions.js | 18 +++++- .../main/webapp/js/nf/canvas/nf-context-menu.js | 5 +- 8 files changed, 162 insertions(+), 11 deletions(-) diff --git a/nifi-docs/src/main/asciidoc/user-guide.adoc b/nifi-docs/src/main/asciidoc/user-guide.adoc index 3024878c2c..9b6286b40f 100644 --- a/nifi-docs/src/main/asciidoc/user-guide.adoc +++ b/nifi-docs/src/main/asciidoc/user-guide.adoc @@ -372,13 +372,15 @@ NOTE: It is also possible to double-click on the Process Group to enter it. - *View connections->Downstream*: This option allows the user to see and "jump to" downstream connections that are going out of the Process Group. - *Center in view*: This option centers the view of the canvas on the given Process Group. - *Group*: This option allows the user to create a new Process Group that contains the selected Process Group and any other components selected on the canvas. -- *Download flow definition*: This option allows the user to download the flow definition of the process group as a JSON file. The file can be used as a backup or imported into a link:https://nifi.apache.org/registry.html[NiFi Registry^] using the <<toolkit-guide.adoc#nifi_CLI,NiFi CLI>>. (Note: If "Download flow definition" is selected for a versioned process group, there is no versioning information in the download. In other words, the resulting contents of the JSON file is the same wh [...] +- *Download flow definition*: This option allows the user to download the flow definition of the process group as a JSON file. The file can be used as a backup or imported into a link:https://nifi.apache.org/registry.html[NiFi Registry^] using the <<toolkit-guide.adoc#nifi_CLI,NiFi CLI>>. There are two options when downloading a flow definition: +** -> *Without external services*: Controller services referenced by the selected process group but outside its scope (e.g., services in a parent group) _will not be_ included in the flow definition as services. +** -> *With external services*: Controller services referenced by the selected process group but outside its scope (e.g., services in a parent group) _will be_ included in the flow definition. - *Create template*: This option allows the user to create a template from the selected Process Group. - *Copy*: This option places a copy of the selected Process Group on the clipboard, so that it may be pasted elsewhere on the canvas by right-clicking on the canvas and selecting Paste. The Copy/Paste actions also may be done using the keystrokes Ctrl-C (Command-C) and Ctrl-V (Command-V). - *Empty all queues*: This option allows the user to empty all queues in the selected Process Group. All FlowFiles from all connections waiting at the time of the request will be removed. - *Delete*: This option allows the DFM to delete a Process Group. - +(Note: If "Download flow definition" is selected for a versioned process group, there is no versioning information in the download. In other words, the resulting contents of the JSON file is the same whether the process group is versioned or not.) [[remote_process_group]] image:iconRemoteProcessGroup.png["Remote Process Group", width=32] diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index 1bbc4d9326..ea78af85ae 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -1624,6 +1624,16 @@ public interface NiFiServiceFacade { */ VersionedFlowSnapshot getCurrentFlowSnapshotByGroupId(String processGroupId); + /** + * Get the current state of the Process Group with the given ID, converted to a Versioned Flow Snapshot. Controller + * Services referenced by the Components contained by the Process Group but are part of the parent Process Group(s) + * will be included and will be considered as part of the requested Process Group. + * + * @param processGroupId the ID of the Process Group + * @return the current Process Group converted to a Versioned Flow Snapshot for download + */ + VersionedFlowSnapshot getCurrentFlowSnapshotByGroupIdWithReferencedControllerServices(String processGroupId); + /** * Returns the name of the Flow Registry that is registered with the given ID. If no Flow Registry exists with the given ID, will return * the ID itself as the name diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index 066b1310f2..4cafa8429f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -4682,6 +4682,23 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { @Override public VersionedFlowSnapshot getCurrentFlowSnapshotByGroupId(final String processGroupId) { + return getCurrentFlowSnapshotByGroupId(processGroupId, false); + } + + @Override + public VersionedFlowSnapshot getCurrentFlowSnapshotByGroupIdWithReferencedControllerServices(final String processGroupId) { + return getCurrentFlowSnapshotByGroupId(processGroupId, true); + } + + private Set<String> getAllSubGroups(ProcessGroup processGroup) { + final Set<String> result = processGroup.findAllProcessGroups().stream() + .map(ProcessGroup::getIdentifier) + .collect(Collectors.toSet()); + result.add(processGroup.getIdentifier()); + return result; + } + + private VersionedFlowSnapshot getCurrentFlowSnapshotByGroupId(final String processGroupId, final boolean includeReferencedControllerServices) { final ProcessGroup processGroup = processGroupDAO.getProcessGroup(processGroupId); // Create a complete (include descendant flows) VersionedProcessGroup snapshot of the flow as it is @@ -4691,12 +4708,40 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { mapper.mapNonVersionedProcessGroup(processGroup, controllerFacade.getControllerServiceProvider()); // Create a complete (include descendant flows) map of parameter contexts - final Map<String, VersionedParameterContext> parameterContexts = - mapper.mapParameterContexts(processGroup, true); + final Map<String, VersionedParameterContext> parameterContexts = mapper.mapParameterContexts(processGroup, true); + final Map<String, ExternalControllerServiceReference> externalControllerServiceReferences = + Optional.ofNullable(nonVersionedProcessGroup.getExternalControllerServiceReferences()).orElse(Collections.emptyMap()); + final Set<VersionedControllerService> controllerServices = new HashSet<>(nonVersionedProcessGroup.getControllerServices()); final VersionedFlowSnapshot nonVersionedFlowSnapshot = new VersionedFlowSnapshot(); + + ProcessGroup parentGroup = processGroup.getParent(); + + if (includeReferencedControllerServices && parentGroup != null) { + final Set<VersionedControllerService> externalServices = new HashSet<>(); + + do { + final Set<ControllerServiceNode> controllerServiceNodes = parentGroup.getControllerServices(false); + + for (final ControllerServiceNode controllerServiceNode : controllerServiceNodes) { + final VersionedControllerService versionedControllerService = + mapper.mapControllerService(controllerServiceNode, controllerFacade.getControllerServiceProvider(), getAllSubGroups(processGroup), externalControllerServiceReferences); + + if (externalControllerServiceReferences.keySet().contains(versionedControllerService.getIdentifier())) { + versionedControllerService.setGroupIdentifier(processGroupId); + externalServices.add(versionedControllerService); + } + } + } while ((parentGroup = parentGroup.getParent()) != null); + + controllerServices.addAll(externalServices); + nonVersionedFlowSnapshot.setExternalControllerServices(new HashMap<>()); + } else { + nonVersionedFlowSnapshot.setExternalControllerServices(externalControllerServiceReferences); + } + + nonVersionedProcessGroup.setControllerServices(controllerServices); nonVersionedFlowSnapshot.setFlowContents(nonVersionedProcessGroup); - nonVersionedFlowSnapshot.setExternalControllerServices(nonVersionedProcessGroup.getExternalControllerServiceReferences()); nonVersionedFlowSnapshot.setParameterContexts(parameterContexts); nonVersionedFlowSnapshot.setFlowEncodingVersion(RestBasedFlowRegistry.FLOW_ENCODING_VERSION); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java index 1c946c72ba..146631613f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java @@ -349,7 +349,18 @@ public class ProcessGroupResource extends FlowUpdateResource<ProcessGroupImportE @ApiResponse(code = 404, message = "The specified resource could not be found."), @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") }) - public Response exportProcessGroup(@ApiParam(value = "The process group id.", required = true) @PathParam("id") final String groupId) { + public Response exportProcessGroup( + @ApiParam( + value = "The process group id.", + required = true + ) + @PathParam("id") final String groupId, + @ApiParam( + value = "If referenced services from outside the target group should be included", + required = false + ) + @QueryParam("includeReferencedServices") + @DefaultValue("false") boolean includeReferencedServices) { // authorize access serviceFacade.authorizeAccess(lookup -> { // ensure access to process groups (nested), encapsulated controller services and referenced parameter contexts @@ -359,7 +370,9 @@ public class ProcessGroupResource extends FlowUpdateResource<ProcessGroupImportE }); // get the versioned flow - final VersionedFlowSnapshot currentVersionedFlowSnapshot = serviceFacade.getCurrentFlowSnapshotByGroupId(groupId); + final VersionedFlowSnapshot currentVersionedFlowSnapshot = includeReferencedServices + ? serviceFacade.getCurrentFlowSnapshotByGroupIdWithReferencedControllerServices(groupId) + : serviceFacade.getCurrentFlowSnapshotByGroupId(groupId); // determine the name of the attachment - possible issues with spaces in file names final VersionedProcessGroup currentVersionedProcessGroup = currentVersionedFlowSnapshot.getFlowContents(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java index 87d80c1a24..50355e69ee 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java @@ -34,7 +34,9 @@ import org.apache.nifi.authorization.user.NiFiUserDetails; import org.apache.nifi.authorization.user.StandardNiFiUser.Builder; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.ControllerServiceProvider; +import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.history.History; @@ -71,10 +73,12 @@ import org.springframework.security.core.context.SecurityContextHolder; import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -84,6 +88,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.anyInt; import static org.mockito.Mockito.mock; @@ -91,6 +96,9 @@ import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.same; +import static org.mockito.Mockito.anySet; +import static org.mockito.Mockito.anyMap; public class StandardNiFiServiceFacadeTest { @@ -381,6 +389,62 @@ public class StandardNiFiServiceFacadeTest { assertNull(versionedFlowSnapshot.getSnapshotMetadata()); } + @Test + public void testGetCurrentFlowSnapshotByGroupIdWithReferencedControllerServices() { + final String groupId = UUID.randomUUID().toString(); + final ProcessGroup processGroup = mock(ProcessGroup.class); + final ProcessGroup parentProcessGroup = mock(ProcessGroup.class); + + final Set<ControllerServiceNode> parentControllerServices = new HashSet<>(); + final ControllerServiceNode parentControllerService1 = mock(ControllerServiceNode.class); + final ControllerServiceNode parentControllerService2 = mock(ControllerServiceNode.class); + parentControllerServices.add(parentControllerService1); + parentControllerServices.add(parentControllerService2); + + when(processGroupDAO.getProcessGroup(groupId)).thenReturn(processGroup); + when(processGroup.getParent()).thenReturn(parentProcessGroup); + when(parentProcessGroup.getControllerServices(anyBoolean())).thenReturn(parentControllerServices); + + final FlowManager flowManager = mock(FlowManager.class); + final ExtensionManager extensionManager = mock(ExtensionManager.class); + when(flowController.getFlowManager()).thenReturn(flowManager); + when(flowController.getExtensionManager()).thenReturn(extensionManager); + + final ControllerServiceProvider controllerServiceProvider = mock(ControllerServiceProvider.class); + when(flowController.getControllerServiceProvider()).thenReturn(controllerServiceProvider); + + final VersionControlInformation versionControlInformation = mock(VersionControlInformation.class); + when(processGroup.getVersionControlInformation()).thenReturn(versionControlInformation); + + // use spy to mock the make() method for generating a new flow mapper to make this testable + final StandardNiFiServiceFacade serviceFacadeSpy = spy(serviceFacade); + final NiFiRegistryFlowMapper flowMapper = mock(NiFiRegistryFlowMapper.class); + when(serviceFacadeSpy.makeNiFiRegistryFlowMapper(extensionManager)).thenReturn(flowMapper); + + final InstantiatedVersionedProcessGroup nonVersionedProcessGroup = spy(new InstantiatedVersionedProcessGroup(UUID.randomUUID().toString(), UUID.randomUUID().toString())); + when(flowMapper.mapNonVersionedProcessGroup(processGroup, controllerServiceProvider)).thenReturn(nonVersionedProcessGroup); + + final VersionedControllerService versionedControllerService1 = mock(VersionedControllerService.class); + final VersionedControllerService versionedControllerService2 = mock(VersionedControllerService.class); + + Mockito.when(versionedControllerService1.getIdentifier()).thenReturn("test"); + Mockito.when(versionedControllerService2.getIdentifier()).thenReturn("test2"); + + when(flowMapper.mapControllerService(same(parentControllerService1), same(controllerServiceProvider), anySet(), anyMap())).thenReturn(versionedControllerService1); + when(flowMapper.mapControllerService(same(parentControllerService2), same(controllerServiceProvider), anySet(), anyMap())).thenReturn(versionedControllerService2); + when(flowMapper.mapParameterContexts(processGroup, true)).thenReturn(new HashMap<>()); + + final ExternalControllerServiceReference externalControllerServiceReference = mock(ExternalControllerServiceReference.class); + final Map<String, ExternalControllerServiceReference> externalControllerServiceReferences = new LinkedHashMap<>(); + externalControllerServiceReferences.put("test", externalControllerServiceReference); + when(nonVersionedProcessGroup.getExternalControllerServiceReferences()).thenReturn(externalControllerServiceReferences); + + final VersionedFlowSnapshot versionedFlowSnapshot = serviceFacadeSpy.getCurrentFlowSnapshotByGroupIdWithReferencedControllerServices(groupId); + + assertEquals(1, versionedFlowSnapshot.getFlowContents().getControllerServices().size()); + assertEquals("test", versionedFlowSnapshot.getFlowContents().getControllerServices().iterator().next().getIdentifier()); + } + @Test public void testIsAnyProcessGroupUnderVersionControl_None() { final String groupId = UUID.randomUUID().toString(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestProcessGroupResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestProcessGroupResource.java index cc74c3dd26..ed28dfe72e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestProcessGroupResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/api/TestProcessGroupResource.java @@ -53,7 +53,7 @@ public class TestProcessGroupResource { when(versionedFlowSnapshot.getFlowContents()).thenReturn(versionedProcessGroup); when(versionedProcessGroup.getName()).thenReturn(flowName); - final Response response = processGroupResource.exportProcessGroup(groupId); + final Response response = processGroupResource.exportProcessGroup(groupId, false); final VersionedFlowSnapshot resultEntity = (VersionedFlowSnapshot)response.getEntity(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js index ae9345dd3d..d256a9c615 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-actions.js @@ -1718,10 +1718,24 @@ } }, + /** + * Downloads the current flow, without including the external Controller Services + */ + downloadFlowWithoutExternalServices: function (selection) { + this.downloadFlow(selection, false); + }, + + /** + * Downloads the current flow, including the external Controller Services + */ + downloadFlowWithExternalServices: function (selection) { + this.downloadFlow(selection, true); + }, + /** * Downloads the current flow */ - downloadFlow: function (selection) { + downloadFlow: function (selection,includeReferencedServices) { var processGroupId = null; if (selection.empty()) { @@ -1737,7 +1751,7 @@ var parameters = {}; // open the url - var uri = '../nifi-api/process-groups/' + encodeURIComponent(processGroupId) + '/download'; + var uri = '../nifi-api/process-groups/' + encodeURIComponent(processGroupId) + '/download?includeReferencedServices=' + includeReferencedServices; window.open(uri); } }, diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js index fd9ec47664..bd0460ecb0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-context-menu.js @@ -861,7 +861,10 @@ {id: 'move-into-parent-menu-item', condition: canMoveToParent, menuItem: {clazz: 'fa fa-arrows', text: 'Move to parent group', action: 'moveIntoParent'}}, {id: 'group-menu-item', condition: canGroup, menuItem: {clazz: 'icon icon-group', text: 'Group', action: 'group'}}, {separator: true}, - {id: 'download-menu-item', condition: supportsDownloadFlow, menuItem: {clazz: 'fa', text: 'Download flow definition', action: 'downloadFlow'}}, + {id: 'download-menu-item', groupMenuItem: {clazz: 'fa', text: 'Download flow definition'}, menuItems: [ + {id: 'download-menu-item-without', condition: hasUpstream, menuItem: {clazz: 'fa', text: 'Without external services', action: 'downloadFlowWithoutExternalServices'}}, + {id: 'download-menu-item-with', condition: hasDownstream, menuItem: {clazz: 'fa', text: 'With external services', action: 'downloadFlowWithExternalServices'}} + ]}, {separator: true}, {id: 'upload-template-menu-item', condition: canUploadTemplate, menuItem: {clazz: 'icon icon-template-import', text: 'Upload template', action: 'uploadTemplate'}}, {id: 'template-menu-item', condition: canCreateTemplate, menuItem: {clazz: 'icon icon-template-save', text: 'Create template', action: 'template'}},
