Repository: incubator-nifi Updated Branches: refs/heads/NIFI-250 fcff5c40a -> a4555d1a1
NIFI-250: Include referenced controller services in templates Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/bd999d16 Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/bd999d16 Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/bd999d16 Branch: refs/heads/NIFI-250 Commit: bd999d16ad804a0c71e6188dac0da57ceda58996 Parents: 24d787e Author: Mark Payne <[email protected]> Authored: Fri Mar 20 14:51:26 2015 -0400 Committer: Mark Payne <[email protected]> Committed: Fri Mar 20 14:51:26 2015 -0400 ---------------------------------------------------------------------- .../apache/nifi/web/api/dto/FlowSnippetDTO.java | 15 +- .../service/ControllerServiceNode.java | 1 - .../apache/nifi/controller/FlowController.java | 37 +++- .../apache/nifi/controller/TemplateManager.java | 33 +++- .../nifi-framework/nifi-nar-utils/.gitignore | 1 + .../nifi/web/api/config/ThrowableMapper.java | 7 +- .../org/apache/nifi/web/api/dto/DtoFactory.java | 19 ++ .../nifi/web/dao/impl/StandardSnippetDAO.java | 36 +++- .../org/apache/nifi/web/util/SnippetUtils.java | 177 ++++++++++++++++++- 9 files changed, 305 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bd999d16/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowSnippetDTO.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowSnippetDTO.java b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowSnippetDTO.java index 61c3c33..47a6871 100644 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowSnippetDTO.java +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowSnippetDTO.java @@ -34,7 +34,8 @@ public class FlowSnippetDTO { private Set<ConnectionDTO> connections = new LinkedHashSet<>(); private Set<LabelDTO> labels = new LinkedHashSet<>(); private Set<FunnelDTO> funnels = new LinkedHashSet<>(); - + private Set<ControllerServiceDTO> controllerServices = new LinkedHashSet<>(); + /** * The connections in this flow snippet. * @@ -138,4 +139,16 @@ public class FlowSnippetDTO { public void setRemoteProcessGroups(Set<RemoteProcessGroupDTO> remoteProcessGroups) { this.remoteProcessGroups = remoteProcessGroups; } + + /** + * Returns the Controller Services in this flow snippet + * @return + */ + public Set<ControllerServiceDTO> getControllerServices() { + return controllerServices; + } + + public void setControllerServices(Set<ControllerServiceDTO> controllerServices) { + this.controllerServices = controllerServices; + } } http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bd999d16/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/service/ControllerServiceNode.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/service/ControllerServiceNode.java b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/service/ControllerServiceNode.java index 68357b8..50bf469 100644 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/service/ControllerServiceNode.java +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/service/ControllerServiceNode.java @@ -18,7 +18,6 @@ package org.apache.nifi.controller.service; import java.util.Set; -import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.controller.ConfiguredComponent; import org.apache.nifi.controller.ControllerService; http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bd999d16/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index fd01711..603ca95 100644 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -176,6 +176,7 @@ import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.ReflectionUtils; import org.apache.nifi.web.api.dto.ConnectableDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; +import org.apache.nifi.web.api.dto.ControllerServiceDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; import org.apache.nifi.web.api.dto.FunnelDTO; import org.apache.nifi.web.api.dto.LabelDTO; @@ -1394,6 +1395,23 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R validateSnippetContents(requireNonNull(group), dto); // + // Instantiate Controller Services + // + for ( final ControllerServiceDTO controllerServiceDTO : dto.getControllerServices() ) { + final ControllerServiceNode serviceNode = createControllerService(controllerServiceDTO.getType(), controllerServiceDTO.getId(), true); + + serviceNode.setAnnotationData(controllerServiceDTO.getAnnotationData()); + serviceNode.setComments(controllerServiceDTO.getComments()); + serviceNode.setName(controllerServiceDTO.getName()); + + for ( final Map.Entry<String, String> entry : controllerServiceDTO.getProperties().entrySet() ) { + if ( entry.getValue() != null ) { + serviceNode.setProperty(entry.getKey(), entry.getValue()); + } + } + } + + // // Instantiate the labels // for (final LabelDTO labelDTO : dto.getLabels()) { @@ -1403,7 +1421,7 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R label.setSize(new Size(labelDTO.getWidth(), labelDTO.getHeight())); } - // TODO: Update the label's "style" + label.setStyle(labelDTO.getStyle()); group.addLabel(label); } @@ -1729,14 +1747,18 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R } // validate that all Processor Types and Prioritizer Types are valid - final List<String> processorClasses = new ArrayList<>(); + final Set<String> processorClasses = new HashSet<>(); for (final Class<?> c : ExtensionManager.getExtensions(Processor.class)) { processorClasses.add(c.getName()); } - final List<String> prioritizerClasses = new ArrayList<>(); + final Set<String> prioritizerClasses = new HashSet<>(); for (final Class<?> c : ExtensionManager.getExtensions(FlowFilePrioritizer.class)) { prioritizerClasses.add(c.getName()); } + final Set<String> controllerServiceClasses = new HashSet<>(); + for (final Class<?> c : ExtensionManager.getExtensions(ControllerService.class)) { + controllerServiceClasses.add(c.getName()); + } final Set<ProcessorDTO> allProcs = new HashSet<>(); final Set<ConnectionDTO> allConns = new HashSet<>(); @@ -1752,6 +1774,15 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R throw new IllegalStateException("Invalid Processor Type: " + proc.getType()); } } + + final Set<ControllerServiceDTO> controllerServices = templateContents.getControllerServices(); + if (controllerServices != null) { + for (final ControllerServiceDTO service : controllerServices) { + if (!controllerServiceClasses.contains(service.getType())) { + throw new IllegalStateException("Invalid Controller Service Type: " + service.getType()); + } + } + } for (final ConnectionDTO conn : allConns) { final List<String> prioritizers = conn.getPrioritizers(); http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bd999d16/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateManager.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateManager.java b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateManager.java index f615f63..30d4365 100644 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateManager.java +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateManager.java @@ -42,24 +42,25 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.persistence.TemplateDeserializer; +import org.apache.nifi.persistence.TemplateSerializer; import org.apache.nifi.stream.io.ByteArrayInputStream; import org.apache.nifi.stream.io.ByteArrayOutputStream; import org.apache.nifi.stream.io.DataOutputStream; import org.apache.nifi.stream.io.StreamUtils; -import org.apache.nifi.persistence.TemplateDeserializer; -import org.apache.nifi.persistence.TemplateSerializer; import org.apache.nifi.web.api.dto.ConnectableDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; +import org.apache.nifi.web.api.dto.ControllerServiceDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; import org.apache.nifi.web.api.dto.ProcessGroupDTO; import org.apache.nifi.web.api.dto.ProcessorConfigDTO; -import org.apache.nifi.web.api.dto.PropertyDescriptorDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; +import org.apache.nifi.web.api.dto.PropertyDescriptorDTO; import org.apache.nifi.web.api.dto.RemoteProcessGroupContentsDTO; import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO; import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO; import org.apache.nifi.web.api.dto.TemplateDTO; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -272,6 +273,11 @@ public class TemplateManager { if (snippet.getProcessGroups() != null) { scrubProcessGroups(snippet.getProcessGroups()); } + + // go through each controller service if specified + if (snippet.getControllerServices() != null) { + scrubControllerServices(snippet.getControllerServices()); + } } } @@ -315,7 +321,6 @@ public class TemplateManager { } } - processorConfig.setDescriptors(null); processorConfig.setCustomUiUrl(null); } @@ -323,6 +328,24 @@ public class TemplateManager { processorDTO.setValidationErrors(null); } } + + private void scrubControllerServices(final Set<ControllerServiceDTO> controllerServices) { + for ( final ControllerServiceDTO serviceDTO : controllerServices ) { + final Map<String, String> properties = serviceDTO.getProperties(); + final Map<String, PropertyDescriptorDTO> descriptors = serviceDTO.getDescriptors(); + + if ( properties != null && descriptors != null ) { + for ( final PropertyDescriptorDTO descriptor : descriptors.values() ) { + if ( descriptor.isSensitive() ) { + properties.put(descriptor.getName(), null); + } + } + } + + serviceDTO.setCustomUiUrl(null); + serviceDTO.setValidationErrors(null); + } + } /** * Scrubs connections prior to saving. This includes removing available http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bd999d16/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/.gitignore ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/.gitignore b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/.gitignore index ea8c4bf..29546b5 100755 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/.gitignore +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/.gitignore @@ -1 +1,2 @@ /target +/target/ http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bd999d16/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/ThrowableMapper.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/ThrowableMapper.java b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/ThrowableMapper.java index 091653a..0ef6edb 100644 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/ThrowableMapper.java +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/config/ThrowableMapper.java @@ -34,11 +34,8 @@ public class ThrowableMapper implements ExceptionMapper<Throwable> { @Override public Response toResponse(Throwable exception) { // log the error - logger.info(String.format("An unexpected error has occurred: %s. Returning %s response.", exception, Response.Status.INTERNAL_SERVER_ERROR)); - - if (logger.isDebugEnabled()) { - logger.debug(StringUtils.EMPTY, exception); - } + logger.error(String.format("An unexpected error has occurred: %s. Returning %s response.", exception, Response.Status.INTERNAL_SERVER_ERROR)); + logger.error(StringUtils.EMPTY, exception); return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity("An unexpected error has occurred. Please check the logs for additional details.").type("text/plain").build(); } http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bd999d16/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java index 7286c83..b2a3094 100644 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java @@ -1964,6 +1964,25 @@ public final class DtoFactory { return copy; } + + public ControllerServiceDTO copy(final ControllerServiceDTO original) { + final ControllerServiceDTO copy = new ControllerServiceDTO(); + copy.setAnnotationData(original.getAnnotationData()); + copy.setAvailability(original.getAvailability()); + copy.setComments(original.getComments()); + copy.setCustomUiUrl(original.getCustomUiUrl()); + copy.setDescriptors(copy(original.getDescriptors())); + copy.setId(original.getId()); + copy.setName(original.getName()); + copy.setProperties(copy(original.getProperties())); + copy.setReferencingComponents(copy(original.getReferencingComponents())); + copy.setState(original.getState()); + copy.setType(original.getType()); + copy.setUri(original.getUri()); + copy.setValidationErrors(copy(original.getValidationErrors())); + return copy; + } + public FunnelDTO copy(final FunnelDTO original) { final FunnelDTO copy = new FunnelDTO(); copy.setId(original.getId()); http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bd999d16/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardSnippetDAO.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardSnippetDAO.java b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardSnippetDAO.java index 92e3a8d..6447464 100644 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardSnippetDAO.java +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardSnippetDAO.java @@ -26,9 +26,11 @@ import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.Snippet; import org.apache.nifi.controller.StandardSnippet; import org.apache.nifi.controller.exception.ProcessorInstantiationException; +import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.web.NiFiCoreException; import org.apache.nifi.web.ResourceNotFoundException; +import org.apache.nifi.web.api.dto.ControllerServiceDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; import org.apache.nifi.web.api.dto.ProcessGroupDTO; import org.apache.nifi.web.api.dto.ProcessorConfigDTO; @@ -36,7 +38,6 @@ import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.api.dto.SnippetDTO; import org.apache.nifi.web.dao.SnippetDAO; import org.apache.nifi.web.util.SnippetUtils; - import org.apache.commons.lang3.StringUtils; /** @@ -285,9 +286,13 @@ public class StandardSnippetDAO implements SnippetDAO { if (snippet != null) { // go through each processor if specified if (snippet.getProcessors() != null) { - lookupSensitiveProperties(snippet.getProcessors()); + lookupSensitiveProcessorProperties(snippet.getProcessors()); } + if ( snippet.getControllerServices() != null ) { + lookupSensitiveControllerServiceProperties(snippet.getControllerServices()); + } + // go through each process group if specified if (snippet.getProcessGroups() != null) { for (final ProcessGroupDTO group : snippet.getProcessGroups()) { @@ -303,7 +308,7 @@ public class StandardSnippetDAO implements SnippetDAO { * * @param snippet */ - private void lookupSensitiveProperties(final Set<ProcessorDTO> processors) { + private void lookupSensitiveProcessorProperties(final Set<ProcessorDTO> processors) { final ProcessGroup rootGroup = flowController.getGroup(flowController.getRootGroupId()); // go through each processor @@ -331,6 +336,31 @@ public class StandardSnippetDAO implements SnippetDAO { } } } + + private void lookupSensitiveControllerServiceProperties(final Set<ControllerServiceDTO> controllerServices) { + // go through each service + for (final ControllerServiceDTO serviceDTO : controllerServices) { + + // ensure that some property configuration have been specified + final Map<String, String> serviceProperties = serviceDTO.getProperties(); + if (serviceProperties != null) { + // find the corresponding controller service + final ControllerServiceNode serviceNode = flowController.getControllerServiceNode(serviceDTO.getId()); + if (serviceNode == null) { + throw new IllegalArgumentException(String.format("Unable to create snippet because Controller Service '%s' could not be found", serviceDTO.getId())); + } + + // look for sensitive properties get the actual value + for (Entry<PropertyDescriptor, String> entry : serviceNode.getProperties().entrySet()) { + final PropertyDescriptor descriptor = entry.getKey(); + + if (descriptor.isSensitive()) { + serviceProperties.put(descriptor.getName(), entry.getValue()); + } + } + } + } + } /* setters */ public void setFlowController(FlowController flowController) { http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/bd999d16/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/SnippetUtils.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/SnippetUtils.java b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/SnippetUtils.java index 8653094..af80539 100644 --- a/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/SnippetUtils.java +++ b/nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/SnippetUtils.java @@ -18,6 +18,7 @@ package org.apache.nifi.web.util; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -28,6 +29,7 @@ import java.util.UUID; import org.apache.nifi.cluster.context.ClusterContext; import org.apache.nifi.cluster.context.ClusterContextThreadLocal; +import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.connectable.ConnectableType; import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.Funnel; @@ -37,17 +39,22 @@ import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ScheduledState; import org.apache.nifi.controller.Snippet; import org.apache.nifi.controller.label.Label; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.controller.service.ControllerServiceState; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.web.api.dto.ConnectableDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; +import org.apache.nifi.web.api.dto.ControllerServiceDTO; import org.apache.nifi.web.api.dto.DtoFactory; import org.apache.nifi.web.api.dto.FlowSnippetDTO; import org.apache.nifi.web.api.dto.FunnelDTO; import org.apache.nifi.web.api.dto.LabelDTO; import org.apache.nifi.web.api.dto.PortDTO; import org.apache.nifi.web.api.dto.ProcessGroupDTO; +import org.apache.nifi.web.api.dto.ProcessorConfigDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; +import org.apache.nifi.web.api.dto.PropertyDescriptorDTO; import org.apache.nifi.web.api.dto.RemoteProcessGroupContentsDTO; import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO; import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO; @@ -181,11 +188,100 @@ public final class SnippetUtils { snippetDto.setRemoteProcessGroups(remoteProcessGroups); } + addControllerServicesToSnippet(snippetDto); + return snippetDto; } + + private void addControllerServicesToSnippet(final FlowSnippetDTO snippetDto) { + for ( final ProcessorDTO processorDto : snippetDto.getProcessors() ) { + addControllerServicesToSnippet(snippetDto, processorDto); + } + + for ( final ProcessGroupDTO processGroupDto : snippetDto.getProcessGroups() ) { + final FlowSnippetDTO childGroupDto = processGroupDto.getContents(); + addControllerServicesToSnippet(childGroupDto); + } + } + + private void addControllerServicesToSnippet(final FlowSnippetDTO snippet, final ProcessorDTO processorDto) { + final ProcessorConfigDTO configDto = processorDto.getConfig(); + if ( configDto == null ) { + return; + } + + final Map<String, PropertyDescriptorDTO> descriptors = configDto.getDescriptors(); + final Map<String, String> properties = configDto.getProperties(); + + if ( properties != null && descriptors != null ) { + for ( final Map.Entry<String, String> entry : properties.entrySet() ) { + final String propName = entry.getKey(); + final String propValue = entry.getValue(); + if ( propValue == null ) { + continue; + } + + final PropertyDescriptorDTO propertyDescriptorDto = descriptors.get(propName); + if ( propertyDescriptorDto != null && propertyDescriptorDto.isIdentifiesControllerService() ) { + final ControllerServiceNode serviceNode = flowController.getControllerServiceNode(propValue); + if ( serviceNode != null ) { + addControllerServicesToSnippet(snippet, serviceNode); + } + } + } + } + } + + private void addControllerServicesToSnippet(final FlowSnippetDTO snippet, final ControllerServiceNode serviceNode) { + if ( isServicePresent(serviceNode.getIdentifier(), snippet.getControllerServices()) ) { + return; + } + + final ControllerServiceDTO serviceNodeDto = dtoFactory.createControllerServiceDto(serviceNode); + Set<ControllerServiceDTO> existingServiceDtos = snippet.getControllerServices(); + if ( existingServiceDtos == null ) { + existingServiceDtos = new HashSet<>(); + snippet.setControllerServices(existingServiceDtos); + } + existingServiceDtos.add(serviceNodeDto); + + for ( final Map.Entry<PropertyDescriptor, String> entry : serviceNode.getProperties().entrySet() ) { + final PropertyDescriptor descriptor = entry.getKey(); + final String propertyValue = entry.getValue(); + + if ( descriptor.getControllerServiceDefinition() != null ) { + final ControllerServiceNode referencedNode = flowController.getControllerServiceNode(propertyValue); + if ( referencedNode == null ) { + throw new IllegalStateException("Controller Service with ID " + propertyValue + " is referenced in template but cannot be found"); + } + + final String referencedNodeId = referencedNode.getIdentifier(); + + final boolean alreadyPresent = isServicePresent(referencedNodeId, snippet.getControllerServices()); + if ( !alreadyPresent ) { + addControllerServicesToSnippet(snippet, referencedNode); + } + } + } + } + private boolean isServicePresent(final String serviceId, final Collection<ControllerServiceDTO> services) { + if ( services == null ) { + return false; + } + + for ( final ControllerServiceDTO existingService : services ) { + if ( serviceId.equals(existingService.getId()) ) { + return true; + } + } + + return false; + } + + public FlowSnippetDTO copy(final FlowSnippetDTO snippetContents, final ProcessGroup group) { - final FlowSnippetDTO snippetCopy = copyContentsForGroup(snippetContents, group.getIdentifier(), null); + final FlowSnippetDTO snippetCopy = copyContentsForGroup(snippetContents, group.getIdentifier(), null, null); resolveNameConflicts(snippetCopy, group); return snippetCopy; } @@ -240,10 +336,45 @@ public final class SnippetUtils { } } - private FlowSnippetDTO copyContentsForGroup(final FlowSnippetDTO snippetContents, final String groupId, final Map<String, ConnectableDTO> parentConnectableMap) { + private FlowSnippetDTO copyContentsForGroup(final FlowSnippetDTO snippetContents, final String groupId, final Map<String, ConnectableDTO> parentConnectableMap, Map<String, String> serviceIdMap) { final FlowSnippetDTO snippetContentsCopy = new FlowSnippetDTO(); // + // Copy the Controller Services + // + if ( serviceIdMap == null ) { + serviceIdMap = new HashMap<>(); + final Set<ControllerServiceDTO> services = new HashSet<>(); + if ( snippetContents.getControllerServices() != null ) { + for (final ControllerServiceDTO serviceDTO : snippetContents.getControllerServices() ) { + final ControllerServiceDTO service = dtoFactory.copy(serviceDTO); + service.setId(generateId(serviceDTO.getId())); + service.setState(ControllerServiceState.DISABLED.name()); + services.add(service); + + // Map old service ID to new service ID so that we can make sure that we reference the new ones. + serviceIdMap.put(serviceDTO.getId(), service.getId()); + } + } + + // if there is any controller service that maps to another controller service, update the id's + for ( final ControllerServiceDTO serviceDTO : services ) { + final Map<String, String> properties = serviceDTO.getProperties(); + final Map<String, PropertyDescriptorDTO> descriptors = serviceDTO.getDescriptors(); + if ( properties != null && descriptors != null ) { + for ( final PropertyDescriptorDTO descriptor : descriptors.values() ) { + if ( descriptor.isIdentifiesControllerService() ) { + final String currentServiceId = properties.get(descriptor.getName()); + final String newServiceId = serviceIdMap.get(currentServiceId); + properties.put(descriptor.getName(), newServiceId); + } + } + } + } + snippetContentsCopy.setControllerServices(services); + } + + // // Copy the labels // final Set<LabelDTO> labels = new HashSet<>(); @@ -332,6 +463,9 @@ public final class SnippetUtils { } snippetContentsCopy.setProcessors(processors); + // if there is any controller service that maps to another controller service, update the id's + updateControllerServiceIdentifiers(snippetContentsCopy, serviceIdMap); + // // Copy ProcessGroups // @@ -344,7 +478,7 @@ public final class SnippetUtils { cp.setParentGroupId(groupId); // copy the contents of this group - we do not copy via the dto factory since we want to specify new ids - final FlowSnippetDTO contentsCopy = copyContentsForGroup(groupDTO.getContents(), cp.getId(), connectableMap); + final FlowSnippetDTO contentsCopy = copyContentsForGroup(groupDTO.getContents(), cp.getId(), connectableMap, serviceIdMap); cp.setContents(contentsCopy); groups.add(cp); } @@ -396,6 +530,43 @@ public final class SnippetUtils { return snippetContentsCopy; } + + + private void updateControllerServiceIdentifiers(final FlowSnippetDTO snippet, final Map<String, String> serviceIdMap) { + final Set<ProcessorDTO> processors = snippet.getProcessors(); + if ( processors != null ) { + for ( final ProcessorDTO processor : processors ) { + updateControllerServiceIdentifiers(processor.getConfig(), serviceIdMap); + } + } + + for ( final ProcessGroupDTO processGroupDto : snippet.getProcessGroups() ) { + updateControllerServiceIdentifiers(processGroupDto.getContents(), serviceIdMap); + } + } + + private void updateControllerServiceIdentifiers(final ProcessorConfigDTO configDto, final Map<String, String> serviceIdMap) { + if ( configDto == null ) { + return; + } + + final Map<String, String> properties = configDto.getProperties(); + final Map<String, PropertyDescriptorDTO> descriptors = configDto.getDescriptors(); + if ( properties != null && descriptors != null ) { + for ( final PropertyDescriptorDTO descriptor : descriptors.values() ) { + if ( descriptor.isIdentifiesControllerService() ) { + final String currentServiceId = properties.get(descriptor.getName()); + if ( currentServiceId == null ) { + continue; + } + + final String newServiceId = serviceIdMap.get(currentServiceId); + properties.put(descriptor.getName(), newServiceId); + } + } + } + } + /** * Generates a new id for the current id that is specified. If no seed is
