Repository: nifi-registry Updated Branches: refs/heads/master cc3820990 -> 44c353396
NIFIREG-62: Added ability to diff two Controller Services without diff'ing entire flow - Use HashSets instead of LinkedHashSets for VersionedProcessGroup, since there is no longer a need for the strict ordering This closes #47. Signed-off-by: Bryan Bende <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi-registry/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-registry/commit/44c35339 Tree: http://git-wip-us.apache.org/repos/asf/nifi-registry/tree/44c35339 Diff: http://git-wip-us.apache.org/repos/asf/nifi-registry/diff/44c35339 Branch: refs/heads/master Commit: 44c3533964a9082becb100d1734739acce14b97e Parents: cc38209 Author: Mark Payne <[email protected]> Authored: Thu Nov 30 14:03:20 2017 -0500 Committer: Bryan Bende <[email protected]> Committed: Mon Dec 4 17:00:24 2017 -0500 ---------------------------------------------------------------------- .../flow/VersionedControllerService.java | 10 +++ .../registry/flow/VersionedProcessGroup.java | 45 +++++------ .../nifi/registry/flow/VersionedProcessor.java | 10 +++ .../flow/VersionedPropertyDescriptor.java | 53 ++++++++++++ .../nifi/registry/flow/diff/FlowComparator.java | 13 +++ .../flow/diff/StandardFlowComparator.java | 85 ++++++++++++++++---- 6 files changed, 177 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-registry/blob/44c35339/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedControllerService.java ---------------------------------------------------------------------- diff --git a/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedControllerService.java b/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedControllerService.java index e0b21da..586b1b9 100644 --- a/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedControllerService.java +++ b/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedControllerService.java @@ -29,6 +29,7 @@ public class VersionedControllerService extends VersionedComponent { private List<ControllerServiceAPI> controllerServiceApis; private Map<String, String> properties; + private Map<String, VersionedPropertyDescriptor> propertyDescriptors; private String annotationData; @@ -68,6 +69,15 @@ public class VersionedControllerService extends VersionedComponent { this.properties = properties; } + @ApiModelProperty("The property descriptors for the processor.") + public Map<String, VersionedPropertyDescriptor> getPropertyDescriptors() { + return propertyDescriptors; + } + + public void setPropertyDescriptors(Map<String, VersionedPropertyDescriptor> propertyDescriptors) { + this.propertyDescriptors = propertyDescriptors; + } + @ApiModelProperty(value = "The annotation for the controller service. This is how the custom UI relays configuration to the controller service.") public String getAnnotationData() { return annotationData; http://git-wip-us.apache.org/repos/asf/nifi-registry/blob/44c35339/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedProcessGroup.java ---------------------------------------------------------------------- diff --git a/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedProcessGroup.java b/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedProcessGroup.java index f20d1ef..2acd0a4 100644 --- a/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedProcessGroup.java +++ b/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedProcessGroup.java @@ -18,29 +18,26 @@ package org.apache.nifi.registry.flow; import java.util.HashMap; -import java.util.LinkedHashSet; +import java.util.HashSet; import java.util.Map; import java.util.Set; -import io.swagger.annotations.ApiModelProperty; - import javax.xml.bind.annotation.XmlRootElement; +import io.swagger.annotations.ApiModelProperty; + @XmlRootElement public class VersionedProcessGroup extends VersionedComponent { - // It is important that all of these components be held in LinkedHashSets because if a VersionedProcessGroup - // is sent to a cluster and all nodes must add more than 1 component, we have to ensure that we add them in the - // same order so that their generated ID's are deterministic. - private LinkedHashSet<VersionedProcessGroup> processGroups = new LinkedHashSet<>(); - private LinkedHashSet<VersionedRemoteProcessGroup> remoteProcessGroups = new LinkedHashSet<>(); - private LinkedHashSet<VersionedProcessor> processors = new LinkedHashSet<>(); - private LinkedHashSet<VersionedPort> inputPorts = new LinkedHashSet<>(); - private LinkedHashSet<VersionedPort> outputPorts = new LinkedHashSet<>(); - private LinkedHashSet<VersionedConnection> connections = new LinkedHashSet<>(); - private LinkedHashSet<VersionedLabel> labels = new LinkedHashSet<>(); - private LinkedHashSet<VersionedFunnel> funnels = new LinkedHashSet<>(); - private LinkedHashSet<VersionedControllerService> controllerServices = new LinkedHashSet<>(); + private Set<VersionedProcessGroup> processGroups = new HashSet<>(); + private Set<VersionedRemoteProcessGroup> remoteProcessGroups = new HashSet<>(); + private Set<VersionedProcessor> processors = new HashSet<>(); + private Set<VersionedPort> inputPorts = new HashSet<>(); + private Set<VersionedPort> outputPorts = new HashSet<>(); + private Set<VersionedConnection> connections = new HashSet<>(); + private Set<VersionedLabel> labels = new HashSet<>(); + private Set<VersionedFunnel> funnels = new HashSet<>(); + private Set<VersionedControllerService> controllerServices = new HashSet<>(); private VersionedFlowCoordinates versionedFlowCoordinates = null; private Map<String, String> variables = new HashMap<>(); @@ -51,7 +48,7 @@ public class VersionedProcessGroup extends VersionedComponent { } public void setProcessGroups(Set<VersionedProcessGroup> processGroups) { - this.processGroups = new LinkedHashSet<>(processGroups); + this.processGroups = new HashSet<>(processGroups); } @ApiModelProperty("The Remote Process Groups") @@ -60,7 +57,7 @@ public class VersionedProcessGroup extends VersionedComponent { } public void setRemoteProcessGroups(Set<VersionedRemoteProcessGroup> remoteProcessGroups) { - this.remoteProcessGroups = new LinkedHashSet<>(remoteProcessGroups); + this.remoteProcessGroups = new HashSet<>(remoteProcessGroups); } @ApiModelProperty("The Processors") @@ -69,7 +66,7 @@ public class VersionedProcessGroup extends VersionedComponent { } public void setProcessors(Set<VersionedProcessor> processors) { - this.processors = new LinkedHashSet<>(processors); + this.processors = new HashSet<>(processors); } @ApiModelProperty("The Input Ports") @@ -78,7 +75,7 @@ public class VersionedProcessGroup extends VersionedComponent { } public void setInputPorts(Set<VersionedPort> inputPorts) { - this.inputPorts = new LinkedHashSet<>(inputPorts); + this.inputPorts = new HashSet<>(inputPorts); } @ApiModelProperty("The Output Ports") @@ -87,7 +84,7 @@ public class VersionedProcessGroup extends VersionedComponent { } public void setOutputPorts(Set<VersionedPort> outputPorts) { - this.outputPorts = new LinkedHashSet<>(outputPorts); + this.outputPorts = new HashSet<>(outputPorts); } @ApiModelProperty("The Connections") @@ -96,7 +93,7 @@ public class VersionedProcessGroup extends VersionedComponent { } public void setConnections(Set<VersionedConnection> connections) { - this.connections = new LinkedHashSet<>(connections); + this.connections = new HashSet<>(connections); } @ApiModelProperty("The Labels") @@ -105,7 +102,7 @@ public class VersionedProcessGroup extends VersionedComponent { } public void setLabels(Set<VersionedLabel> labels) { - this.labels = new LinkedHashSet<>(labels); + this.labels = new HashSet<>(labels); } @ApiModelProperty("The Funnels") @@ -114,7 +111,7 @@ public class VersionedProcessGroup extends VersionedComponent { } public void setFunnels(Set<VersionedFunnel> funnels) { - this.funnels = new LinkedHashSet<>(funnels); + this.funnels = new HashSet<>(funnels); } @ApiModelProperty("The Controller Services") @@ -123,7 +120,7 @@ public class VersionedProcessGroup extends VersionedComponent { } public void setControllerServices(Set<VersionedControllerService> controllerServices) { - this.controllerServices = new LinkedHashSet<>(controllerServices); + this.controllerServices = new HashSet<>(controllerServices); } @Override http://git-wip-us.apache.org/repos/asf/nifi-registry/blob/44c35339/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedProcessor.java ---------------------------------------------------------------------- diff --git a/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedProcessor.java b/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedProcessor.java index 314875c..3063066 100644 --- a/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedProcessor.java +++ b/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedProcessor.java @@ -29,6 +29,7 @@ public class VersionedProcessor extends VersionedComponent { private String type; private Map<String, String> properties; + private Map<String, VersionedPropertyDescriptor> propertyDescriptors; private String annotationData; private String schedulingPeriod; @@ -124,6 +125,15 @@ public class VersionedProcessor extends VersionedComponent { this.properties = properties; } + @ApiModelProperty("The property descriptors for the processor.") + public Map<String, VersionedPropertyDescriptor> getPropertyDescriptors() { + return propertyDescriptors; + } + + public void setPropertyDescriptors(Map<String, VersionedPropertyDescriptor> propertyDescriptors) { + this.propertyDescriptors = propertyDescriptors; + } + @ApiModelProperty("The annotation data for the processor used to relay configuration between a custom UI and the procesosr.") public String getAnnotationData() { return annotationData; http://git-wip-us.apache.org/repos/asf/nifi-registry/blob/44c35339/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedPropertyDescriptor.java ---------------------------------------------------------------------- diff --git a/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedPropertyDescriptor.java b/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedPropertyDescriptor.java new file mode 100644 index 0000000..ebb560a --- /dev/null +++ b/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/flow/VersionedPropertyDescriptor.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.registry.flow; + +import io.swagger.annotations.ApiModelProperty; + +public class VersionedPropertyDescriptor { + private String name; + private String displayName; + private boolean identifiesControllerService; + + @ApiModelProperty("The name of the property") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @ApiModelProperty("The display name of the property") + public String getDisplayName() { + return displayName; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + @ApiModelProperty("Whether or not the property provides the identifier of a Controller Service") + public boolean getIdentifiesControllerService() { + return identifiesControllerService; + } + + public void setIdentifiesControllerService(boolean identifiesControllerService) { + this.identifiesControllerService = identifiesControllerService; + } +} http://git-wip-us.apache.org/repos/asf/nifi-registry/blob/44c35339/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/FlowComparator.java ---------------------------------------------------------------------- diff --git a/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/FlowComparator.java b/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/FlowComparator.java index 94e1d8c..3835fec 100644 --- a/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/FlowComparator.java +++ b/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/FlowComparator.java @@ -17,6 +17,19 @@ package org.apache.nifi.registry.flow.diff; +import java.util.Set; + +import org.apache.nifi.registry.flow.VersionedControllerService; + public interface FlowComparator { FlowComparison compare(); + + /** + * Compares to versions of a Controller Service and returns the differences between them + * + * @param serviceA the first Controller Service + * @param serviceB the second Controller Service + * @return the differences between them + */ + Set<FlowDifference> compareControllerServices(VersionedControllerService serviceA, VersionedControllerService serviceB); } http://git-wip-us.apache.org/repos/asf/nifi-registry/blob/44c35339/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java ---------------------------------------------------------------------- diff --git a/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java b/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java index 260909e..3541ab5 100644 --- a/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java +++ b/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java @@ -34,6 +34,7 @@ import org.apache.nifi.registry.flow.VersionedLabel; import org.apache.nifi.registry.flow.VersionedPort; import org.apache.nifi.registry.flow.VersionedProcessGroup; import org.apache.nifi.registry.flow.VersionedProcessor; +import org.apache.nifi.registry.flow.VersionedPropertyDescriptor; import org.apache.nifi.registry.flow.VersionedRemoteGroupPort; import org.apache.nifi.registry.flow.VersionedRemoteProcessGroup; @@ -41,11 +42,14 @@ public class StandardFlowComparator implements FlowComparator { private final ComparableDataFlow flowA; private final ComparableDataFlow flowB; + private final Set<String> externallyAccessibleServiceIds; private final DifferenceDescriptor differenceDescriptor; - public StandardFlowComparator(final ComparableDataFlow flowA, final ComparableDataFlow flowB, final DifferenceDescriptor differenceDescriptor) { + public StandardFlowComparator(final ComparableDataFlow flowA, final ComparableDataFlow flowB, + final Set<String> externallyAccessibleServiceIds, final DifferenceDescriptor differenceDescriptor) { this.flowA = flowA; this.flowB = flowB; + this.externallyAccessibleServiceIds = externallyAccessibleServiceIds; this.differenceDescriptor = differenceDescriptor; } @@ -99,11 +103,11 @@ public class StandardFlowComparator implements FlowComparator { private boolean compareComponents(final VersionedComponent componentA, final VersionedComponent componentB, final Set<FlowDifference> differences) { - return compareComponents(componentA, componentB, differences, true, true); + return compareComponents(componentA, componentB, differences, true, true, true); } - private boolean compareComponents(final VersionedComponent componentA, final VersionedComponent componentB, final Set<FlowDifference> differences, final boolean compareNamePos, - final boolean compareComments) { + private boolean compareComponents(final VersionedComponent componentA, final VersionedComponent componentB, final Set<FlowDifference> differences, + final boolean compareName, final boolean comparePos, final boolean compareComments) { if (componentA == null) { differences.add(difference(DifferenceType.COMPONENT_ADDED, componentA, componentB, componentA, componentB)); return true; @@ -115,11 +119,14 @@ public class StandardFlowComparator implements FlowComparator { } if (compareComments) { - addIfDifferent(differences, DifferenceType.COMMENTS_CHANGED, componentA, componentB, VersionedComponent::getComments); + addIfDifferent(differences, DifferenceType.COMMENTS_CHANGED, componentA, componentB, VersionedComponent::getComments, false); } - if (compareNamePos) { + if (compareName) { addIfDifferent(differences, DifferenceType.NAME_CHANGED, componentA, componentB, VersionedComponent::getName); + } + + if (comparePos) { addIfDifferent(differences, DifferenceType.POSITION_CHANGED, componentA, componentB, VersionedComponent::getPosition); } @@ -143,7 +150,14 @@ public class StandardFlowComparator implements FlowComparator { addIfDifferent(differences, DifferenceType.SCHEDULING_STRATEGY_CHANGED, processorA, processorB, VersionedProcessor::getSchedulingStrategy); addIfDifferent(differences, DifferenceType.STYLE_CHANGED, processorA, processorB, VersionedProcessor::getStyle); addIfDifferent(differences, DifferenceType.YIELD_DURATION_CHANGED, processorA, processorB, VersionedProcessor::getYieldDuration); - compareProperties(processorA, processorB, processorA.getProperties(), processorB.getProperties(), differences); + compareProperties(processorA, processorB, processorA.getProperties(), processorB.getProperties(), processorA.getPropertyDescriptors(), differences); + } + + @Override + public Set<FlowDifference> compareControllerServices(final VersionedControllerService serviceA, final VersionedControllerService serviceB) { + final Set<FlowDifference> differences = new HashSet<>(); + compare(serviceA, serviceB, differences); + return differences; } private void compare(final VersionedControllerService serviceA, final VersionedControllerService serviceB, final Set<FlowDifference> differences) { @@ -153,24 +167,43 @@ public class StandardFlowComparator implements FlowComparator { addIfDifferent(differences, DifferenceType.ANNOTATION_DATA_CHANGED, serviceA, serviceB, VersionedControllerService::getAnnotationData); addIfDifferent(differences, DifferenceType.BUNDLE_CHANGED, serviceA, serviceB, VersionedControllerService::getBundle); - compareProperties(serviceA, serviceB, serviceA.getProperties(), serviceB.getProperties(), differences); + compareProperties(serviceA, serviceB, serviceA.getProperties(), serviceB.getProperties(), serviceA.getPropertyDescriptors(), differences); } private void compareProperties(final VersionedComponent componentA, final VersionedComponent componentB, - final Map<String, String> propertiesA, final Map<String, String> propertiesB, final Set<FlowDifference> differences) { + final Map<String, String> propertiesA, final Map<String, String> propertiesB, final Map<String, VersionedPropertyDescriptor> descriptorsA, + final Set<FlowDifference> differences) { propertiesA.entrySet().stream() .forEach(entry -> { final String valueA = entry.getValue(); final String valueB = propertiesB.get(entry.getKey()); + final VersionedPropertyDescriptor descriptor = descriptorsA.get(entry.getKey()); + final String displayName = descriptor.getDisplayName() == null ? descriptor.getName() : descriptor.getDisplayName(); + if (valueA == null && valueB != null) { - differences.add(difference(DifferenceType.PROPERTY_ADDED, componentA, componentB, entry.getKey(), entry.getKey())); + differences.add(difference(DifferenceType.PROPERTY_ADDED, componentA, componentB, displayName, displayName)); } else if (valueA != null && valueB == null) { - differences.add(difference(DifferenceType.PROPERTY_REMOVED, componentA, componentB, entry.getKey(), entry.getKey())); + differences.add(difference(DifferenceType.PROPERTY_REMOVED, componentA, componentB, displayName, displayName)); } else if (valueA != null && valueB != null && !valueA.equals(valueB)) { - differences.add(difference(DifferenceType.PROPERTY_CHANGED, componentA, componentB, valueA, valueB)); + // If the property in Flow A references a Controller Service that is not available in the flow + // and the property in Flow B references a Controller Service that is available in its environment + // but not part of the Versioned Flow, then we do not want to consider this to be a Flow Difference. + // This is typically the case when a flow is versioned in one instance, referencing an external Controller Service, + // and then imported into another NiFi instance. When imported, the property does not point to any existing Controller + // Service, and the user must then point the property an existing Controller Service. We don't want to consider the + // flow as having changed, since it is an environment-specific change (similar to how we handle variables). + if (descriptor.getIdentifiesControllerService()) { + final boolean accessibleA = externallyAccessibleServiceIds.contains(valueA); + final boolean accessibleB = externallyAccessibleServiceIds.contains(valueB); + if (!accessibleA && accessibleB) { + return; + } + } + + differences.add(difference(DifferenceType.PROPERTY_CHANGED, componentA, componentB, displayName + "=" + valueA, displayName + "=" + valueB)); } }); @@ -211,7 +244,7 @@ public class StandardFlowComparator implements FlowComparator { } private void compare(final VersionedRemoteProcessGroup rpgA, final VersionedRemoteProcessGroup rpgB, final Set<FlowDifference> differences) { - if (compareComponents(rpgA, rpgB, differences, true, false)) { // do not compare comments for RPG because they come from remote system, not our local flow + if (compareComponents(rpgA, rpgB, differences, false, true, false)) { // do not compare comments for RPG because they come from remote system, not our local flow return; } @@ -239,7 +272,7 @@ public class StandardFlowComparator implements FlowComparator { private void compare(final VersionedProcessGroup groupA, final VersionedProcessGroup groupB, final Set<FlowDifference> differences, final boolean compareNamePos) { - if (compareComponents(groupA, groupB, differences, compareNamePos, true)) { + if (compareComponents(groupA, groupB, differences, compareNamePos, compareNamePos, true)) { return; } @@ -287,10 +320,15 @@ public class StandardFlowComparator implements FlowComparator { return components.stream().collect(Collectors.toMap(VersionedComponent::getIdentifier, Function.identity())); } - private <T extends VersionedComponent> void addIfDifferent(final Set<FlowDifference> differences, final DifferenceType type, final T componentA, final T componentB, final Function<T, Object> transform) { + addIfDifferent(differences, type, componentA, componentB, transform, true); + } + + private <T extends VersionedComponent> void addIfDifferent(final Set<FlowDifference> differences, final DifferenceType type, final T componentA, final T componentB, + final Function<T, Object> transform, final boolean differentiateNullAndEmptyString) { + final Object valueA = transform.apply(componentA); final Object valueB = transform.apply(componentB); @@ -303,6 +341,10 @@ public class StandardFlowComparator implements FlowComparator { return; } + if (!differentiateNullAndEmptyString && isEmptyString(valueA) && isEmptyString(valueB)) { + return; + } + differences.add(difference(type, componentA, componentB, valueA, valueB)); } @@ -310,6 +352,19 @@ public class StandardFlowComparator implements FlowComparator { return collection == null || collection.isEmpty(); } + private boolean isEmptyString(final Object potentialString) { + if (potentialString == null) { + return true; + } + + if (potentialString instanceof String) { + final String string = (String) potentialString; + return string.isEmpty(); + } else { + return false; + } + } + private FlowDifference difference(final DifferenceType type, final VersionedComponent componentA, final VersionedComponent componentB, final Object valueA, final Object valueB) { final String description = differenceDescriptor.describeDifference(type, flowA.getName(), flowB.getName(), componentA, componentB, valueA, valueB); return new StandardFlowDifference(type, componentA, componentB, valueA, valueB, description);
