Repository: nifi Updated Branches: refs/heads/master 8412d2662 -> 1bf10944e
NIFI-2366 - Fixed ID generation semantics in clustered environment - added SnippetUtilsTest - renamed TypeOneUUIDGenerator to ComponentIdGenerator - changed lsb part of ComponentIdGenerator back to long - Fixed 'isCopy' condition for clustered environments This closes #718. Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/1bf10944 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/1bf10944 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/1bf10944 Branch: refs/heads/master Commit: 1bf10944eaee0e408d870c9e17a6c8224b630fad Parents: 8412d26 Author: Oleg Zhurakousky <[email protected]> Authored: Mon Jul 25 16:28:38 2016 -0400 Committer: Mark Payne <[email protected]> Committed: Sun Jul 31 15:24:02 2016 -0400 ---------------------------------------------------------------------- .../apache/nifi/util/ComponentIdGenerator.java | 102 +++++++++ .../apache/nifi/util/TypeOneUUIDGenerator.java | 76 ------- .../apache/nifi/web/api/dto/ComponentDTO.java | 5 + .../apache/nifi/web/api/dto/FlowSnippetDTO.java | 68 ++++++ .../ThreadPoolRequestReplicator.java | 4 +- .../apache/nifi/controller/FlowController.java | 6 +- .../apache/nifi/controller/TemplateUtils.java | 42 ---- .../persistence/TemplateSerializerTest.java | 6 +- .../nifi/web/api/ApplicationResource.java | 7 +- .../org/apache/nifi/web/util/SnippetUtils.java | 72 ++++-- .../apache/nifi/web/util/SnippetUtilsTest.java | 219 +++++++++++++++++++ 11 files changed, 460 insertions(+), 147 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/ComponentIdGenerator.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/ComponentIdGenerator.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/ComponentIdGenerator.java new file mode 100644 index 0000000..49f00b8 --- /dev/null +++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/ComponentIdGenerator.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.util; + +import java.security.SecureRandom; +import java.util.UUID; + +/** + * IMPORTANT: This component is not part of public API! + * ==================================================== + * <p> + * This component generates type-one UUID. It is used for generating ID of all + * NiFi components. Giving the 128-bit UUID structure which consists of Least + * Significant Bits (LSB) and Most Significant Bits (MSB) this component + * provides support for generating and maintaining INCEPTION ID of the component + * (MSB) as well as its INSTANCE ID (LSB). + * </p> + * <p> + * It is also important to understand that while this component does seed itself + * from current time which could be extracted from the resulting UUID via call + * to {@link UUID#timestamp()} operation, one should not be relying on such time + * as the exact time when such ID was generated since in the event the same time + * is passed to one of the {@link #generateId()} operation it will be + * incremented by 1 since the goal of this component to only ensure uniqueness + * and type-one semantics where each UUID generated by this component is + * comparable and each subsequent ID is > then previous ID. + * </p> + * <p> + * For more details on how it is interacted with please see + * org.apache.nifi.web.util.SnippetUtils as well as + * org.apache.nifi.web.util.SnippetUtilsTest which contain additional + * documentation on its usage as well as ID generation contracts defined in + * NiFi. + * </p> + */ +public class ComponentIdGenerator { + + public static final Object lock = new Object(); + + private static long lastTime; + private static long clockSequence = 0; + private static final SecureRandom randomGenerator = new SecureRandom(); + + /** + * Will generate unique time based UUID where the next UUID is always + * greater then the previous. + */ + public final static UUID generateId() { + return generateId(System.currentTimeMillis()); + } + + /** + * + */ + public final static UUID generateId(long currentTime) { + return generateId(currentTime, randomGenerator.nextLong(), true); + } + + /** + * + */ + public final static UUID generateId(long msb, long lsb, boolean ensureUnique) { + long time; + + synchronized (lock) { + if (ensureUnique && msb <= lastTime) { + msb = ++lastTime; + } + lastTime = msb; + } + + time = msb; + + // low Time + time = msb << 32; + + // mid Time + time |= ((msb & 0xFFFF00000000L) >> 16); + + // hi Time + time |= 0x1000 | ((msb >> 48) & 0x0FFF); + + long clockSequenceHi = clockSequence; + clockSequenceHi <<= 48; + lsb = clockSequenceHi | lsb; + return new UUID(time, lsb); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/TypeOneUUIDGenerator.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/TypeOneUUIDGenerator.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/TypeOneUUIDGenerator.java deleted file mode 100644 index 5d8ee22..0000000 --- a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/TypeOneUUIDGenerator.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.nifi.util; - -import java.util.Random; -import java.util.UUID; - -public class TypeOneUUIDGenerator { - - public static final Object lock = new Object(); - - private static long lastTime; - private static long clockSequence = 0; - private static final Random randomGenerator = new Random(); - - /** - * Will generate unique time based UUID where the next UUID is always - * greater then the previous. - */ - public final static UUID generateId() { - return generateId(System.currentTimeMillis()); - } - - /** - * - */ - public final static UUID generateId(long currentTime) { - return generateId(currentTime, Math.abs(randomGenerator.nextInt())); - } - - /** - * - */ - public final static UUID generateId(long currentTime, int lsbInt) { - long time; - - synchronized (lock) { - if (currentTime == lastTime) { - ++clockSequence; - } else { - lastTime = currentTime; - clockSequence = 0; - } - } - - time = currentTime; - - // low Time - time = currentTime << 32; - - // mid Time - time |= ((currentTime & 0xFFFF00000000L) >> 16); - - // hi Time - time |= 0x1000 | ((currentTime >> 48) & 0x0FFF); - - long clockSequenceHi = clockSequence; - clockSequenceHi <<= 48; - long lsb = clockSequenceHi | lsbInt; - return new UUID(time, lsb); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ComponentDTO.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ComponentDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ComponentDTO.java index f85adf5..dfbca3d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ComponentDTO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ComponentDTO.java @@ -93,4 +93,9 @@ public class ComponentDTO { return id.equals(((ComponentDTO) obj).getId()); } + + @Override + public String toString() { + return this.getClass().getSimpleName() + ":" + this.getId(); + } } http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowSnippetDTO.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowSnippetDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowSnippetDTO.java index 63194a3..f268134 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowSnippetDTO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowSnippetDTO.java @@ -203,19 +203,87 @@ public class FlowSnippetDTO { UUID id = UUID.fromString(componentDto.getId()); id = new UUID(id.getMostSignificantBits(), 0); componentDto.setId(id.toString()); + + id = UUID.fromString(componentDto.getParentGroupId()); + id = new UUID(id.getMostSignificantBits(), 0); + componentDto.setParentGroupId(id.toString()); + if (componentDto instanceof ConnectionDTO) { ConnectionDTO connectionDTO = (ConnectionDTO) componentDto; + ConnectableDTO cdto = connectionDTO.getSource(); id = UUID.fromString(cdto.getId()); id = new UUID(id.getMostSignificantBits(), 0); cdto.setId(id.toString()); + id = UUID.fromString(cdto.getGroupId()); + id = new UUID(id.getMostSignificantBits(), 0); + cdto.setGroupId(id.toString()); + cdto = connectionDTO.getDestination(); id = UUID.fromString(cdto.getId()); id = new UUID(id.getMostSignificantBits(), 0); cdto.setId(id.toString()); + + id = UUID.fromString(cdto.getGroupId()); + id = new UUID(id.getMostSignificantBits(), 0); + cdto.setGroupId(id.toString()); + } + if (componentDto instanceof ProcessGroupDTO) { + FlowSnippetDTO fsDTO = ((ProcessGroupDTO) componentDto).getContents(); + + this.removeInstanceIdentifierIfNecessary(fsDTO.getConnections()); + fsDTO.connections = this.orderedById(fsDTO.getConnections()); + + this.removeInstanceIdentifierIfNecessary(fsDTO.getControllerServices()); + fsDTO.controllerServices = this.orderedById(fsDTO.getControllerServices()); + + this.removeInstanceIdentifierIfNecessary(fsDTO.getFunnels()); + fsDTO.funnels = this.orderedById(fsDTO.getFunnels()); + + this.removeInstanceIdentifierIfNecessary(fsDTO.getInputPorts()); + fsDTO.inputPorts = this.orderedById(fsDTO.getInputPorts()); + + this.removeInstanceIdentifierIfNecessary(fsDTO.getLabels()); + fsDTO.labels = this.orderedById(fsDTO.getLabels()); + + this.removeInstanceIdentifierIfNecessary(fsDTO.getOutputPorts()); + fsDTO.outputPorts = this.orderedById(fsDTO.getOutputPorts()); + + this.removeInstanceIdentifierIfNecessary(fsDTO.getProcessGroups()); + fsDTO.processGroups = this.orderedById(fsDTO.getProcessGroups()); + + this.removeInstanceIdentifierIfNecessary(fsDTO.getProcessors()); + fsDTO.processors = this.orderedById(fsDTO.getProcessors()); + + this.removeInstanceIdentifierIfNecessary(fsDTO.getRemoteProcessGroups()); + fsDTO.remoteProcessGroups = this.orderedById(fsDTO.getRemoteProcessGroups()); + } else if (componentDto instanceof RemoteProcessGroupDTO) { + RemoteProcessGroupContentsDTO contentsDTO = ((RemoteProcessGroupDTO) componentDto).getContents(); + for (RemoteProcessGroupPortDTO portDTO : contentsDTO.getInputPorts()) { + id = UUID.fromString(portDTO.getId()); + id = new UUID(id.getMostSignificantBits(), 0); + portDTO.setId(new UUID(id.getMostSignificantBits(), 0).toString()); + } + for (RemoteProcessGroupPortDTO portDTO : contentsDTO.getOutputPorts()) { + id = UUID.fromString(portDTO.getId()); + portDTO.setId(new UUID(id.getMostSignificantBits(), 0).toString()); + } + contentsDTO.setInputPorts(this.orderedRemotePortsById(contentsDTO.getInputPorts())); + contentsDTO.setOutputPorts(this.orderedRemotePortsById(contentsDTO.getOutputPorts())); } } } } + + private <T extends RemoteProcessGroupPortDTO> Set<T> orderedRemotePortsById(Set<T> dtos) { + TreeSet<T> components = new TreeSet<>(new Comparator<RemoteProcessGroupPortDTO>() { + @Override + public int compare(RemoteProcessGroupPortDTO c1, RemoteProcessGroupPortDTO c2) { + return UUID.fromString(c1.getId()).compareTo(UUID.fromString(c2.getId())); + } + }); + components.addAll(dtos); + return components; + } } http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ThreadPoolRequestReplicator.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ThreadPoolRequestReplicator.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ThreadPoolRequestReplicator.java index c5a8af5..f4d4611 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ThreadPoolRequestReplicator.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/replication/ThreadPoolRequestReplicator.java @@ -41,7 +41,7 @@ import org.apache.nifi.cluster.protocol.NodeIdentifier; import org.apache.nifi.events.EventReporter; import org.apache.nifi.reporting.Severity; import org.apache.nifi.util.FormatUtils; -import org.apache.nifi.util.TypeOneUUIDGenerator; +import org.apache.nifi.util.ComponentIdGenerator; import org.apache.nifi.web.security.ProxiedEntitiesUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -219,7 +219,7 @@ public class ThreadPoolRequestReplicator implements RequestReplicator { final boolean indicateReplicated, final boolean performVerification) { final Map<String, String> updatedHeaders = new HashMap<>(headers); - updatedHeaders.put(RequestReplicator.CLUSTER_ID_GENERATION_SEED_HEADER, TypeOneUUIDGenerator.generateId().toString()); + updatedHeaders.put(RequestReplicator.CLUSTER_ID_GENERATION_SEED_HEADER, ComponentIdGenerator.generateId().toString()); if (indicateReplicated) { updatedHeaders.put(RequestReplicator.REPLICATION_INDICATOR_HEADER, "true"); } http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 1b14cf8..8879726 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -186,6 +186,7 @@ import org.apache.nifi.stream.io.StreamUtils; import org.apache.nifi.util.FormatUtils; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.ReflectionUtils; +import org.apache.nifi.util.ComponentIdGenerator; import org.apache.nifi.web.ResourceNotFoundException; import org.apache.nifi.web.api.dto.ConnectableDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; @@ -515,9 +516,10 @@ public class FlowController implements EventAccess, ControllerServiceProvider, R this.snippetManager = new SnippetManager(); - rootGroup = new StandardProcessGroup(UUID.randomUUID().toString(), this, processScheduler, properties, encryptor, this, this.variableRegistry); + rootGroup = new StandardProcessGroup(ComponentIdGenerator.generateId().toString(), this, processScheduler, + properties, encryptor, this, this.variableRegistry); rootGroup.setName(DEFAULT_ROOT_GROUP_NAME); - instanceId = UUID.randomUUID().toString(); + instanceId = ComponentIdGenerator.generateId().toString(); controllerServiceProvider = new StandardControllerServiceProvider(this, processScheduler, bulletinRepository, stateManagerProvider, this.variableRegistry); http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateUtils.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateUtils.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateUtils.java index a5c4679..21dba95 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateUtils.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/TemplateUtils.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -43,9 +42,7 @@ import org.apache.nifi.web.api.dto.ProcessorConfigDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.api.dto.PropertyDescriptorDTO; import org.apache.nifi.web.api.dto.RelationshipDTO; -import org.apache.nifi.web.api.dto.RemoteProcessGroupContentsDTO; import org.apache.nifi.web.api.dto.RemoteProcessGroupDTO; -import org.apache.nifi.web.api.dto.RemoteProcessGroupPortDTO; import org.apache.nifi.web.api.dto.TemplateDTO; import org.w3c.dom.Element; @@ -282,45 +279,6 @@ public class TemplateUtils { remoteProcessGroupDTO.setName(null); remoteProcessGroupDTO.setTargetSecure(null); remoteProcessGroupDTO.setTransmitting(null); - - // if this remote process group has contents - if (remoteProcessGroupDTO.getContents() != null) { - RemoteProcessGroupContentsDTO contents = remoteProcessGroupDTO.getContents(); - - // scrub any remote input ports - if (contents.getInputPorts() != null) { - scrubRemotePorts(contents.getInputPorts()); - } - - // scrub and remote output ports - if (contents.getOutputPorts() != null) { - scrubRemotePorts(contents.getOutputPorts()); - } - } - } - } - - /** - * Remove unnecessary fields in remote ports prior to saving. - * - * @param remotePorts ports - */ - private static void scrubRemotePorts(final Set<RemoteProcessGroupPortDTO> remotePorts) { - for (final Iterator<RemoteProcessGroupPortDTO> remotePortIter = remotePorts.iterator(); remotePortIter.hasNext();) { - final RemoteProcessGroupPortDTO remotePortDTO = remotePortIter.next(); - - // if the flow is not connected to this remote port, remove it - if (remotePortDTO.isConnected() == null || !remotePortDTO.isConnected().booleanValue()) { - remotePortIter.remove(); - continue; - } - - remotePortDTO.setExists(null); - remotePortDTO.setTargetRunning(null); - remotePortDTO.setConnected(null); - remotePortDTO.setExists(null); - remotePortDTO.setTargetRunning(null); - remotePortDTO.setTransmitting(null); } } } http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TemplateSerializerTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TemplateSerializerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TemplateSerializerTest.java index e5acdfe..8ddfdfe 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TemplateSerializerTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/persistence/TemplateSerializerTest.java @@ -33,7 +33,7 @@ import javax.xml.bind.JAXBElement; import javax.xml.bind.Unmarshaller; import javax.xml.transform.stream.StreamSource; -import org.apache.nifi.util.TypeOneUUIDGenerator; +import org.apache.nifi.util.ComponentIdGenerator; import org.apache.nifi.web.api.dto.FlowSnippetDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; import org.apache.nifi.web.api.dto.TemplateDTO; @@ -55,7 +55,7 @@ public class TemplateSerializerTest { for (int i = 4; i > 0; i--) { ProcessorDTO procDTO = new ProcessorDTO(); procDTO.setType("Processor" + i + ".class"); - procDTO.setId(TypeOneUUIDGenerator.generateId().toString()); + procDTO.setId(ComponentIdGenerator.generateId().toString()); procs.add(procDTO); } snippet.setProcessors(procs); @@ -86,7 +86,7 @@ public class TemplateSerializerTest { // add new Processor ProcessorDTO procDTO = new ProcessorDTO(); procDTO.setType("ProcessorNew" + ".class"); - procDTO.setId(TypeOneUUIDGenerator.generateId().toString()); + procDTO.setId(ComponentIdGenerator.generateId().toString()); deserProcs.add(procDTO); // Serialize modified template http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java index 6a9e1d0..2c5b43e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java @@ -41,7 +41,7 @@ import org.apache.nifi.remote.exception.NotAuthorizedException; import org.apache.nifi.remote.protocol.ResponseCode; import org.apache.nifi.remote.protocol.http.HttpHeaders; import org.apache.nifi.util.NiFiProperties; -import org.apache.nifi.util.TypeOneUUIDGenerator; +import org.apache.nifi.util.ComponentIdGenerator; import org.apache.nifi.authorization.AuthorizableLookup; import org.apache.nifi.authorization.AuthorizeAccess; import org.apache.nifi.web.NiFiServiceFacade; @@ -205,14 +205,15 @@ public abstract class ApplicationResource { if (seed.isPresent()) { try { UUID seedId = UUID.fromString(seed.get()); - uuid = new UUID(seedId.getMostSignificantBits(), Math.abs(seed.get().hashCode())); + uuid = new UUID(seedId.getMostSignificantBits(), seed.get().hashCode()); } catch (Exception e) { logger.warn("Provided 'seed' does not represent UUID. Will not be able to extract most significant bits for ID generation."); uuid = UUID.nameUUIDFromBytes(seed.get().getBytes(StandardCharsets.UTF_8)); } } else { - uuid = TypeOneUUIDGenerator.generateId(); + uuid = ComponentIdGenerator.generateId(); } + return uuid.toString(); } http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/SnippetUtils.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/SnippetUtils.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/SnippetUtils.java index f88889e..18aeca8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/SnippetUtils.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/SnippetUtils.java @@ -16,6 +16,22 @@ */ package org.apache.nifi.web.util; + + +import java.nio.charset.StandardCharsets; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + import org.apache.commons.lang3.StringUtils; import org.apache.nifi.authorization.AccessPolicy; import org.apache.nifi.authorization.RequestAction; @@ -36,7 +52,7 @@ import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.ControllerServiceState; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.groups.RemoteProcessGroup; -import org.apache.nifi.util.TypeOneUUIDGenerator; +import org.apache.nifi.util.ComponentIdGenerator; import org.apache.nifi.web.api.dto.AccessPolicyDTO; import org.apache.nifi.web.api.dto.ConnectableDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; @@ -58,19 +74,6 @@ import org.apache.nifi.web.dao.AccessPolicyDAO; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; - /** * Template utilities. */ @@ -78,6 +81,8 @@ public final class SnippetUtils { private static final Logger logger = LoggerFactory.getLogger(SnippetUtils.class); + private static final SecureRandom randomGenerator = new SecureRandom(); + private FlowController flowController; private DtoFactory dtoFactory; private AccessPolicyDAO accessPolicyDAO; @@ -771,15 +776,44 @@ public final class SnippetUtils { } /** - * Generates a new id for the current id that is specified. If no seed is found, a new random id will be created. + * Generates a new type 1 id (UUID) for the current id that is specified. If + * seed is provided, it will be incorporated into generation logic of the + * new ID. + * The contract of this method is as follows: + * - The 'currentId' must never be null and it must be String representation + * of type-one UUID. + * - If seed is provided, the new ID will be generated from the 'msb' extracted from + * the 'currentId' and the 'lsb' extracted from the UUID generated via + * UUID.nameUUIDFromBytes(currentId + seed). + * - If seed is NOT provided and 'isCopy' flag is set the new ID will be generated from + * the 'msb' extracted from the 'currentId' and random integer as 'lsb'. In this case + * the new ID will always be > the previous ID essentially resulting in the new ID for + * the component that being copied (e.g., copy/paste). + * - If seed is NOT provided and 'isCopy' flag is NOT set the new ID will be generated from + * the 'msb' extracted from the 'currentId' and random integer as 'lsb'. */ private String generateId(final String currentId, final String seed, boolean isCopy) { long msb = UUID.fromString(currentId).getMostSignificantBits(); - int lsb = StringUtils.isBlank(seed) - ? Math.abs(new Random().nextInt()) - : Math.abs(seed.hashCode()); - return isCopy ? TypeOneUUIDGenerator.generateId(msb, lsb).toString() : new UUID(msb, lsb).toString(); + UUID uuid; + if (StringUtils.isBlank(seed)) { + long lsb = randomGenerator.nextLong(); + if (isCopy) { + uuid = ComponentIdGenerator.generateId(msb, lsb, true); // will increment msb if necessary + } else { + // since msb is extracted from type-one UUID, the type-one semantics will be preserved + uuid = new UUID(msb, lsb); + } + } else { + UUID seedId = UUID.nameUUIDFromBytes((currentId + seed).getBytes(StandardCharsets.UTF_8)); + if (isCopy) { + // will ensure the type-one semantics for new UUID generated from msb extracted from seedId + uuid = ComponentIdGenerator.generateId(seedId.getMostSignificantBits(), seedId.getLeastSignificantBits(), false); + } else { + uuid = new UUID(msb, seedId.getLeastSignificantBits()); + } + } + return uuid.toString(); } /* setters */ http://git-wip-us.apache.org/repos/asf/nifi/blob/1bf10944/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/util/SnippetUtilsTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/util/SnippetUtilsTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/util/SnippetUtilsTest.java new file mode 100644 index 0000000..0c169b0 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/util/SnippetUtilsTest.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import org.apache.nifi.util.ComponentIdGenerator; +import org.junit.Test; + +public class SnippetUtilsTest { + + /* + * This test validates condition where component is being replicated across + * the cluster + */ + @Test + public void validateWithSameSeedSameInceptionIdSameInstanceId() throws Exception { + Method generateIdMethod = SnippetUtils.class.getDeclaredMethod("generateId", String.class, String.class, + boolean.class); + generateIdMethod.setAccessible(true); + + SnippetUtils utils = new SnippetUtils(); + String currentId = ComponentIdGenerator.generateId().toString(); + String seed = ComponentIdGenerator.generateId().toString(); + String id1 = (String) generateIdMethod.invoke(utils, currentId, seed, true); + String id2 = (String) generateIdMethod.invoke(utils, currentId, seed, true); + String id3 = (String) generateIdMethod.invoke(utils, currentId, seed, true); + assertEquals(id1, id2); + assertEquals(id2, id3); + } + + /* + * This test validates condition where components that are being copy/pasted from + * one another are now replicated across the cluster. Such components will + * have different inception id (msb) yet different instance id (lsb). The id + * of these components must be different yet their msb must be the same. + */ + @Test + public void validateWithSameSeedSameInceptionIdNotSameInstanceIdIsCopySet() throws Exception { + Method generateIdMethod = SnippetUtils.class.getDeclaredMethod("generateId", String.class, String.class, + boolean.class); + generateIdMethod.setAccessible(true); + + SnippetUtils utils = new SnippetUtils(); + String seed = ComponentIdGenerator.generateId().toString(); + + UUID rootId = ComponentIdGenerator.generateId(); + + String id1 = (String) generateIdMethod.invoke(utils, + new UUID(rootId.getMostSignificantBits(), ComponentIdGenerator.generateId().getLeastSignificantBits()).toString(), + seed, true); + String id2 = (String) generateIdMethod.invoke(utils, + new UUID(rootId.getMostSignificantBits(), ComponentIdGenerator.generateId().getLeastSignificantBits()).toString(), + seed, true); + String id3 = (String) generateIdMethod.invoke(utils, + new UUID(rootId.getMostSignificantBits(), ComponentIdGenerator.generateId().getLeastSignificantBits()).toString(), + seed, true); + assertNotEquals(id1, id2); + assertNotEquals(id2, id3); + UUID uuid1 = UUID.fromString(id1); + UUID uuid2 = UUID.fromString(id2); + UUID uuid3 = UUID.fromString(id3); + // below simply validates that generated UUID is type-one, since timestamp() operation will result + // in exception if generated UUID is not type-one + uuid1.timestamp(); + uuid2.timestamp(); + uuid3.timestamp(); + assertNotEquals(uuid1.getMostSignificantBits(), uuid2.getMostSignificantBits()); + assertNotEquals(uuid2.getMostSignificantBits(), uuid3.getMostSignificantBits()); + } + + /* + * This test validates condition where components that are being re-created + * from template are now replicated across the cluster. Such components will + * have the same inception id (msb) yet different instance id (lsb). The id + * of these components must be different yet their msb must be the same. + */ + @Test + public void validateWithSameSeedSameInceptionIdNotSameInstanceIdIsCopyNotSet() throws Exception { + Method generateIdMethod = SnippetUtils.class.getDeclaredMethod("generateId", String.class, String.class, + boolean.class); + generateIdMethod.setAccessible(true); + + SnippetUtils utils = new SnippetUtils(); + String seed = ComponentIdGenerator.generateId().toString(); + + UUID rootId = ComponentIdGenerator.generateId(); + + String id1 = (String) generateIdMethod.invoke(utils, + new UUID(rootId.getMostSignificantBits(), ComponentIdGenerator.generateId().getLeastSignificantBits()).toString(), + seed, false); + String id2 = (String) generateIdMethod.invoke(utils, + new UUID(rootId.getMostSignificantBits(), ComponentIdGenerator.generateId().getLeastSignificantBits()).toString(), + seed, false); + String id3 = (String) generateIdMethod.invoke(utils, + new UUID(rootId.getMostSignificantBits(), ComponentIdGenerator.generateId().getLeastSignificantBits()).toString(), + seed, false); + assertNotEquals(id1, id2); + assertNotEquals(id2, id3); + UUID uuid1 = UUID.fromString(id1); + UUID uuid2 = UUID.fromString(id2); + UUID uuid3 = UUID.fromString(id3); + // below simply validates that generated UUID is type-one, since timestamp() operation will result + // in exception if generated UUID is not type-one + uuid1.timestamp(); + uuid2.timestamp(); + uuid3.timestamp(); + assertEquals(uuid1.getMostSignificantBits(), uuid2.getMostSignificantBits()); + assertEquals(uuid2.getMostSignificantBits(), uuid3.getMostSignificantBits()); + } + + /* + * This test validates condition where components are being copied from one + * another. The ids of each components must be completely different (msb and + * lsb) yet each subsequent msb must be > then previous component's msb. + */ + @Test + public void validateWithoutSeedSameCurrentIdIsCopySet() throws Exception { + Method generateIdMethod = SnippetUtils.class.getDeclaredMethod("generateId", String.class, String.class, + boolean.class); + generateIdMethod.setAccessible(true); + + boolean isCopy = true; + + SnippetUtils utils = new SnippetUtils(); + String currentId = ComponentIdGenerator.generateId().toString(); + UUID id1 = UUID.fromString((String) generateIdMethod.invoke(utils, currentId, null, isCopy)); + UUID id2 = UUID.fromString((String) generateIdMethod.invoke(utils, currentId, null, isCopy)); + UUID id3 = UUID.fromString((String) generateIdMethod.invoke(utils, currentId, null, isCopy)); + // below simply validates that generated UUID is type-one, since timestamp() operation will result + // in exception if generated UUID is not type-one + id1.timestamp(); + id2.timestamp(); + id3.timestamp(); + assertTrue(id1.getMostSignificantBits() < id2.getMostSignificantBits()); + assertTrue(id2.getMostSignificantBits() < id3.getMostSignificantBits()); + } + + /* + * This test validates condition where new components are being created from + * existing components such as from imported templates. In this case their + * instance id (lsb) is irrelevant and new instance id would have to be + * generated every time yet its inception id (msb) must remain the same. + */ + @Test + public void validateWithoutSeedSameCurrentIdIsCopyNotSet() throws Exception { + Method generateIdMethod = SnippetUtils.class.getDeclaredMethod("generateId", String.class, String.class, + boolean.class); + generateIdMethod.setAccessible(true); + + boolean isCopy = false; + + SnippetUtils utils = new SnippetUtils(); + String currentId = ComponentIdGenerator.generateId().toString(); + UUID id1 = UUID.fromString((String) generateIdMethod.invoke(utils, currentId, null, isCopy)); + UUID id2 = UUID.fromString((String) generateIdMethod.invoke(utils, currentId, null, isCopy)); + UUID id3 = UUID.fromString((String) generateIdMethod.invoke(utils, currentId, null, isCopy)); + // below simply validates that generated UUID is type-one, since timestamp() operation will result + // in exception if generated UUID is not type-one + id1.timestamp(); + id2.timestamp(); + id3.timestamp(); + assertEquals(id1.getMostSignificantBits(), id2.getMostSignificantBits()); + assertEquals(id2.getMostSignificantBits(), id3.getMostSignificantBits()); + } + + /* + * This test simply validates that generated IDs are comparable and + * sequentially correct where each subsequent ID is > previous ID. + */ + @Test + public void validateIdOrdering() throws Exception { + UUID seed = ComponentIdGenerator.generateId(); + UUID currentId1 = ComponentIdGenerator.generateId(); + UUID currentId2 = ComponentIdGenerator.generateId(); + UUID currentId3 = ComponentIdGenerator.generateId(); + + UUID id1 = new UUID(currentId1.getMostSignificantBits(), + UUID.nameUUIDFromBytes((currentId1.toString() + seed.toString()).getBytes(StandardCharsets.UTF_8)) + .getLeastSignificantBits()); + UUID id2 = new UUID(currentId2.getMostSignificantBits(), + UUID.nameUUIDFromBytes((currentId2.toString() + seed.toString()).getBytes(StandardCharsets.UTF_8)) + .getLeastSignificantBits()); + UUID id3 = new UUID(currentId3.getMostSignificantBits(), + UUID.nameUUIDFromBytes((currentId3.toString() + seed.toString()).getBytes(StandardCharsets.UTF_8)) + .getLeastSignificantBits()); + List<UUID> list = new ArrayList<>(); + list.add(id2); + list.add(id3); + list.add(id1); + Collections.sort(list); + assertEquals(id1, list.get(0)); + assertEquals(id2, list.get(1)); + assertEquals(id3, list.get(2)); + } +}
