Repository: nifi-minifi Updated Branches: refs/heads/master 65cd7c590 -> 71a316ffc
MINIFI-482 Provide support for multiple URIs. This closes #148. Signed-off-by: Aldrin Piri <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi/commit/71a316ff Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi/tree/71a316ff Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi/diff/71a316ff Branch: refs/heads/master Commit: 71a316ffc138e6bd25f340fd47c502bbb21effd8 Parents: 65cd7c5 Author: Aldrin Piri <[email protected]> Authored: Tue Dec 4 17:27:57 2018 -0500 Committer: Aldrin Piri <[email protected]> Committed: Wed Dec 12 10:16:01 2018 -0500 ---------------------------------------------------------------------- .../bootstrap/util/ConfigTransformer.java | 5 +- .../bootstrap/util/ConfigTransformerTest.java | 2 +- .../schema/RemoteProcessGroupSchema.java | 12 +- .../dto/RemoteProcessGroupSchemaFunction.java | 8 +- .../toolkit/configuration/ConfigMainTest.java | 5 + .../dto/RemoteProcessGroupSchemaTest.java | 2 +- .../src/test/resources/MultipleUriRPG.xml | 215 +++++++++++++++++++ .../src/test/resources/MultipleUriRPG.yml | 123 +++++++++++ 8 files changed, 363 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/71a316ff/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java index 0da92ac..90b944e 100644 --- a/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java +++ b/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java @@ -68,6 +68,7 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -554,7 +555,9 @@ public final class ConfigTransformer { addTextElement(element, "name", remoteProcessGroupProperties.getName()); addPosition(element); addTextElement(element, "comment", remoteProcessGroupProperties.getComment()); - addTextElement(element, "url", remoteProcessGroupProperties.getUrl()); + // In the case we have multiple urls, select the first + addTextElement(element, "url", Arrays.asList(remoteProcessGroupProperties.getUrls().split(",")).get(0)); + addTextElement(element, "urls", remoteProcessGroupProperties.getUrls()); addTextElement(element, "timeout", remoteProcessGroupProperties.getTimeout()); addTextElement(element, "yieldPeriod", remoteProcessGroupProperties.getYieldPeriod()); addTextElement(element, "transmitting", "true"); http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/71a316ff/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java ---------------------------------------------------------------------- diff --git a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java index 7d2d48b..3f251cb 100644 --- a/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java +++ b/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java @@ -538,7 +538,7 @@ public class ConfigTransformerTest { assertEquals(remoteProcessingGroupSchema.getId(), getText(element, "id")); assertEquals(remoteProcessingGroupSchema.getName(), getText(element, "name")); assertEquals(remoteProcessingGroupSchema.getComment(), getText(element, "comment")); - assertEquals(remoteProcessingGroupSchema.getUrl(), getText(element, "url")); + assertEquals(remoteProcessingGroupSchema.getUrls(), getText(element, "url")); assertEquals(remoteProcessingGroupSchema.getTimeout(), getText(element, "timeout")); assertEquals(remoteProcessingGroupSchema.getYieldPeriod(), getText(element, "yieldPeriod")); assertEquals(remoteProcessingGroupSchema.getTransportProtocol(), getText(element, "transportProtocol")); http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/71a316ff/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java ---------------------------------------------------------------------- diff --git a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java index c22deff..686f476 100644 --- a/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java +++ b/minifi-commons/minifi-commons-schema/src/main/java/org/apache/nifi/minifi/commons/schema/RemoteProcessGroupSchema.java @@ -63,7 +63,7 @@ public class RemoteProcessGroupSchema extends BaseSchemaWithIdAndName { public static final String DEFAULT_PROXY_PASSWORD = ""; public static final String DEFAULT_NETWORK_INTERFACE = ""; - private String url; + private final String urls; private List<RemotePortSchema> inputPorts; private List<RemotePortSchema> outputPorts; @@ -80,7 +80,9 @@ public class RemoteProcessGroupSchema extends BaseSchemaWithIdAndName { public RemoteProcessGroupSchema(Map map) { super(map, "RemoteProcessGroup(id: {id}, name: {name})"); String wrapperName = getWrapperName(); - url = getRequiredKeyAsType(map, URL_KEY, String.class, wrapperName); + // This is either a singular URL or a comma separated list + urls = getRequiredKeyAsType(map, URL_KEY, String.class, wrapperName); + inputPorts = convertListToType(getOptionalKeyAsType(map, INPUT_PORTS_KEY, List.class, wrapperName, new ArrayList<>()), "input port", RemotePortSchema.class, INPUT_PORTS_KEY); addIssuesIfNotNull(inputPorts); @@ -132,7 +134,7 @@ public class RemoteProcessGroupSchema extends BaseSchemaWithIdAndName { @Override public Map<String, Object> toMap() { Map<String, Object> result = super.toMap(); - result.put(URL_KEY, url); + result.put(URL_KEY, urls); result.put(COMMENT_KEY, comment); result.put(TIMEOUT_KEY, timeout); result.put(YIELD_PERIOD_KEY, yieldPeriod); @@ -151,8 +153,8 @@ public class RemoteProcessGroupSchema extends BaseSchemaWithIdAndName { return comment; } - public String getUrl() { - return url; + public String getUrls() { + return urls; } public String getTimeout() { http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/71a316ff/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java index e429e56..03fbf5e 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaFunction.java @@ -17,6 +17,7 @@ package org.apache.nifi.minifi.toolkit.configuration.dto; +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.minifi.commons.schema.RemotePortSchema; import org.apache.nifi.minifi.commons.schema.RemoteProcessGroupSchema; import org.apache.nifi.minifi.commons.schema.common.CommonPropertyKeys; @@ -42,7 +43,12 @@ public class RemoteProcessGroupSchemaFunction implements Function<RemoteProcessG Map<String, Object> map = new HashMap<>(); map.put(CommonPropertyKeys.ID_KEY, remoteProcessGroupDTO.getId()); map.put(CommonPropertyKeys.NAME_KEY, remoteProcessGroupDTO.getName()); - map.put(RemoteProcessGroupSchema.URL_KEY, remoteProcessGroupDTO.getTargetUri()); + + + // Prefer the targetUris if populated, otherwise, default to using the singular targetUri + final String targetUris = remoteProcessGroupDTO.getTargetUris(); + map.put(RemoteProcessGroupSchema.URL_KEY, + StringUtils.isNotBlank(targetUris) ? targetUris : remoteProcessGroupDTO.getTargetUri()); RemoteProcessGroupContentsDTO contents = remoteProcessGroupDTO.getContents(); if (contents != null) { http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/71a316ff/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java index ab940cf..af454bb 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java @@ -222,6 +222,11 @@ public class ConfigMainTest { } @Test + public void testTransformRoundTripMultipleUriRPG() throws IOException, JAXBException, SchemaLoaderException { + transformRoundTrip("MultipleUriRPG"); + } + + @Test public void testSuccessTransformProcessGroup() throws IOException, JAXBException, SchemaLoaderException { ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("TemplateWithProcessGroup.xml")).toMap(); } http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/71a316ff/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaTest.java ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaTest.java index 30dd5ae..4d1539e 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/dto/RemoteProcessGroupSchemaTest.java @@ -155,7 +155,7 @@ public class RemoteProcessGroupSchemaTest extends BaseSchemaTester<RemoteProcess @Override public void assertSchemaEquals(RemoteProcessGroupSchema one, RemoteProcessGroupSchema two) { assertEquals(one.getName(), two.getName()); - assertEquals(one.getUrl(), two.getUrl()); + assertEquals(one.getUrls(), two.getUrls()); List<RemotePortSchema> oneInputPorts = one.getInputPorts(); List<RemotePortSchema> twoInputPorts = two.getInputPorts(); http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/71a316ff/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.xml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.xml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.xml new file mode 100644 index 0000000..887812f --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.xml @@ -0,0 +1,215 @@ +<?xml version="1.0" ?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + --> +<template encoding-version="1.0"> + <description></description> + <groupId>92d57ac5-0158-1000-aab7-42ca398aadea</groupId> + <name>Simple TailFile To RPG</name> + <snippet> + <connections> + <id>b0c0c3cc-0158-1000-0000-000000000000</id> + <parentGroupId>92d57ac5-0158-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>b0c09ff0-0158-1000-0000-000000000000</groupId> + <id>ac0e798c-0158-1000-0588-cda9b944e011</id> + <type>REMOTE_INPUT_PORT</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>92d57ac5-0158-1000-0000-000000000000</groupId> + <id>b0c04f28-0158-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <processors> + <id>b0c04f28-0158-1000-0000-000000000000</id> + <parentGroupId>92d57ac5-0158-1000-0000-000000000000</parentGroupId> + <position> + <x>20.698639744598154</x> + <y>0.0</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>tail-mode</key> + <value> + <name>tail-mode</name> + </value> + </entry> + <entry> + <key>File to Tail</key> + <value> + <name>File to Tail</name> + </value> + </entry> + <entry> + <key>Rolling Filename Pattern</key> + <value> + <name>Rolling Filename Pattern</name> + </value> + </entry> + <entry> + <key>tail-base-directory</key> + <value> + <name>tail-base-directory</name> + </value> + </entry> + <entry> + <key>Initial Start Position</key> + <value> + <name>Initial Start Position</name> + </value> + </entry> + <entry> + <key>File Location</key> + <value> + <name>File Location</name> + </value> + </entry> + <entry> + <key>tailfile-recursive-lookup</key> + <value> + <name>tailfile-recursive-lookup</name> + </value> + </entry> + <entry> + <key>tailfile-rolling-strategy</key> + <value> + <name>tailfile-rolling-strategy</name> + </value> + </entry> + <entry> + <key>tailfile-lookup-frequency</key> + <value> + <name>tailfile-lookup-frequency</name> + </value> + </entry> + <entry> + <key>tailfile-maximum-age</key> + <value> + <name>tailfile-maximum-age</name> + </value> + </entry> + </descriptors> + <executionNode>ALL</executionNode> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>tail-mode</key> + <value>Single file</value> + </entry> + <entry> + <key>File to Tail</key> + <value>./logs/minifi-app.log</value> + </entry> + <entry> + <key>Rolling Filename Pattern</key> + </entry> + <entry> + <key>tail-base-directory</key> + </entry> + <entry> + <key>Initial Start Position</key> + <value>Beginning of File</value> + </entry> + <entry> + <key>File Location</key> + <value>Local</value> + </entry> + <entry> + <key>tailfile-recursive-lookup</key> + <value>false</value> + </entry> + <entry> + <key>tailfile-rolling-strategy</key> + <value>Fixed name</value> + </entry> + <entry> + <key>tailfile-lookup-frequency</key> + <value>10 minutes</value> + </entry> + <entry> + <key>tailfile-maximum-age</key> + <value>24 hours</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>1 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>TailFile</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.TailFile</type> + </processors> + <remoteProcessGroups> + <id>b0c09ff0-0158-1000-0000-000000000000</id> + <parentGroupId>92d57ac5-0158-1000-0000-000000000000</parentGroupId> + <position> + <x>0.0</x> + <y>255.68887264079288</y> + </position> + <communicationsTimeout>30 sec</communicationsTimeout> + <contents> + <inputPorts> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <connected>false</connected> + <exists>true</exists> + <id>aca664f8-0158-1000-a139-92485891d349</id> + <name>test2</name> + <targetRunning>false</targetRunning> + <transmitting>false</transmitting> + <useCompression>false</useCompression> + </inputPorts> + <inputPorts> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <connected>true</connected> + <exists>true</exists> + <id>ac0e798c-0158-1000-0588-cda9b944e011</id> + <name>test</name> + <targetRunning>false</targetRunning> + <transmitting>false</transmitting> + <useCompression>false</useCompression> + </inputPorts> + </contents> + <proxyHost></proxyHost> + <proxyUser></proxyUser> + <targetUri>http://localhost:8080/nifi</targetUri> + <targetUris>http://localhost:8080/nifi,http://localhost:8081/nifi,http://localhost:8082/nifi</targetUris> + <transportProtocol>RAW</transportProtocol> + <yieldDuration>10 sec</yieldDuration> + <networkInterface></networkInterface> + </remoteProcessGroups> + </snippet> + <timestamp>11/29/2016 10:43:33 EST</timestamp> +</template> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/71a316ff/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.yml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.yml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.yml new file mode 100644 index 0000000..f2cfeeb --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/MultipleUriRPG.yml @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 3 +Flow Controller: + name: Simple TailFile To RPG + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 + variable registry properties: '' +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min + implementation: org.apache.nifi.provenance.MiNiFiPersistentProvenanceRepository +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: b0c04f28-0158-1000-0000-000000000000 + name: TailFile + class: org.apache.nifi.processors.standard.TailFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 1 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + File Location: Local + File to Tail: ./logs/minifi-app.log + Initial Start Position: Beginning of File + Rolling Filename Pattern: + tail-base-directory: + tail-mode: Single file + tailfile-lookup-frequency: 10 minutes + tailfile-maximum-age: 24 hours + tailfile-recursive-lookup: 'false' + tailfile-rolling-strategy: Fixed name +Controller Services: [] +Process Groups: [] +Input Ports: [] +Output Ports: [] +Funnels: [] +Connections: +- id: b0c0c3cc-0158-1000-0000-000000000000 + name: TailFile/success/ac0e798c-0158-1000-0588-cda9b944e011 + source id: b0c04f28-0158-1000-0000-000000000000 + source relationship names: + - success + destination id: ac0e798c-0158-1000-0588-cda9b944e011 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +Remote Process Groups: +- id: b0c09ff0-0158-1000-0000-000000000000 + name: '' + url: http://localhost:8080/nifi,http://localhost:8081/nifi,http://localhost:8082/nifi + comment: '' + timeout: 30 sec + yield period: 10 sec + transport protocol: RAW + proxy host: '' + proxy port: '' + proxy user: '' + proxy password: '' + local network interface: '' + Input Ports: + - id: aca664f8-0158-1000-a139-92485891d349 + name: test2 + comment: '' + max concurrent tasks: 1 + use compression: false + - id: ac0e798c-0158-1000-0588-cda9b944e011 + name: test + comment: '' + max concurrent tasks: 1 + use compression: false + Output Ports: [] +NiFi Properties Overrides: {}
