Repository: nifi-minifi Updated Branches: refs/heads/master bd842d339 -> e6f6b0665
MINIFI-86 Adding explicit checks for any unsupported components when transforming a template This closes #38. Signed-off-by: Aldrin Piri <ald...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/nifi-minifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-minifi/commit/e6f6b066 Tree: http://git-wip-us.apache.org/repos/asf/nifi-minifi/tree/e6f6b066 Diff: http://git-wip-us.apache.org/repos/asf/nifi-minifi/diff/e6f6b066 Branch: refs/heads/master Commit: e6f6b0665ace4f13cffa5fc36b4ad48ff02a8647 Parents: bd842d3 Author: Joseph Percivall <joeperciv...@yahoo.com> Authored: Wed Sep 21 19:22:38 2016 -0400 Committer: Aldrin Piri <ald...@apache.org> Committed: Thu Sep 22 20:39:22 2016 -0400 ---------------------------------------------------------------------- .../minifi-toolkit-assembly/README.md | 2 ++ .../toolkit/configuration/ConfigMain.java | 27 +++++++++++++++++--- .../toolkit/configuration/ConfigMainTest.java | 22 +++++++++++++++- .../src/test/resources/TemplateWithFunnel.xml | 18 +++++++++++++ .../test/resources/TemplateWithInputPort.xml | 18 +++++++++++++ .../test/resources/TemplateWithOutputPort.xml | 18 +++++++++++++ .../test/resources/TemplateWithProcessGroup.xml | 19 ++++++++++++++ 7 files changed, 120 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/e6f6b066/minifi-toolkit/minifi-toolkit-assembly/README.md ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-assembly/README.md b/minifi-toolkit/minifi-toolkit-assembly/README.md index c4a49a6..50a7a9b 100644 --- a/minifi-toolkit/minifi-toolkit-assembly/README.md +++ b/minifi-toolkit/minifi-toolkit-assembly/README.md @@ -49,6 +49,8 @@ After downloading the binary and extracting it, to run the MiNiFi Toolkit Conver transform: Transform template xml into MiNiFi config YAML validate: Validate config YAML +Note: Currently MiNiFi does not support Process Groups, Funnels, Input Ports or Output Ports. Any templates that contain these will fail transformation. + ## Getting Help If you have questions, you can reach out to our mailing list: d...@nifi.apache.org ([archive](http://mail-archives.apache.org/mod_mbox/nifi-dev)). http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/e6f6b066/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java index 7cfff75..e44b5d7 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/main/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMain.java @@ -54,7 +54,7 @@ public class ConfigMain { public static final int ERR_UNABLE_TO_OPEN_OUTPUT = 2; public static final int ERR_UNABLE_TO_OPEN_INPUT = 3; public static final int ERR_UNABLE_TO_READ_TEMPLATE = 4; - public static final int ERR_UNABLE_TO_TRANFORM_TEMPLATE = 5; + public static final int ERR_UNABLE_TO_TRANSFORM_TEMPLATE = 5; public static final int ERR_UNABLE_TO_PARSE_CONFIG = 6; public static final int ERR_INVALID_CONFIG = 7; @@ -186,9 +186,26 @@ public class ConfigMain { } } - public static ConfigSchema transformTemplateToSchema(InputStream source) throws JAXBException, IOException { + public static ConfigSchema transformTemplateToSchema(InputStream source) throws JAXBException, IOException, SchemaLoaderException { try { TemplateDTO templateDTO = (TemplateDTO) JAXBContext.newInstance(TemplateDTO.class).createUnmarshaller().unmarshal(source); + + if (templateDTO.getSnippet().getProcessGroups().size() != 0){ + throw new SchemaLoaderException("Process Groups are not currently supported in MiNiFi. Please remove any from the template and try again."); + } + + if (templateDTO.getSnippet().getOutputPorts().size() != 0){ + throw new SchemaLoaderException("Output Ports are not currently supported in MiNiFi. Please remove any from the template and try again."); + } + + if (templateDTO.getSnippet().getInputPorts().size() != 0){ + throw new SchemaLoaderException("Input Ports are not currently supported in MiNiFi. Please remove any from the template and try again."); + } + + if (templateDTO.getSnippet().getFunnels().size() != 0){ + throw new SchemaLoaderException("Funnels are not currently supported in MiNiFi. Please remove any from the template and try again."); + } + enrichTemplateDTO(templateDTO); ConfigSchema configSchema = new ConfigSchemaFunction().apply(templateDTO); return configSchema; @@ -250,7 +267,11 @@ public class ConfigMain { System.out.println("Error transforming template to YAML. (" + e + ")"); System.out.println(); printTransformUsage(); - return ERR_UNABLE_TO_TRANFORM_TEMPLATE; + return ERR_UNABLE_TO_TRANSFORM_TEMPLATE; + } catch (SchemaLoaderException e) { + System.out.println("Error transforming template to YAML. (" + e.getMessage() + ")"); + System.out.println(); + return ERR_UNABLE_TO_TRANSFORM_TEMPLATE; } } catch (FileNotFoundException e) { System.out.println("Unable to open file " + args[2] + " for writing. (" + e + ")"); http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/e6f6b066/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java index bb33d8b..3e21e76 100644 --- a/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/java/org/apache/nifi/minifi/toolkit/configuration/ConfigMainTest.java @@ -137,7 +137,7 @@ public class ConfigMainTest { throw new IOException(); } }); - assertEquals(ConfigMain.ERR_UNABLE_TO_TRANFORM_TEMPLATE, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); + assertEquals(ConfigMain.ERR_UNABLE_TO_TRANSFORM_TEMPLATE, configMain.execute(new String[]{ConfigMain.TRANSFORM, testInput, testOutput})); } @Test @@ -173,6 +173,26 @@ public class ConfigMainTest { transformRoundTrip("StressTestFramework"); } + @Test(expected = SchemaLoaderException.class) + public void testFailToTransformProcessGroup() throws IOException, JAXBException, SchemaLoaderException { + ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("TemplateWithProcessGroup.xml")).toMap(); + } + + @Test(expected = SchemaLoaderException.class) + public void testFailToTransformInputPort() throws IOException, JAXBException, SchemaLoaderException { + ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("TemplateWithOutputPort.xml")).toMap(); + } + + @Test(expected = SchemaLoaderException.class) + public void testFailToTransformOutputPort() throws IOException, JAXBException, SchemaLoaderException { + ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("TemplateWithInputPort.xml")).toMap(); + } + + @Test(expected = SchemaLoaderException.class) + public void testFailToTransformFunnel() throws IOException, JAXBException, SchemaLoaderException { + ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream("TemplateWithFunnel.xml")).toMap(); + } + private void transformRoundTrip(String name) throws JAXBException, IOException, SchemaLoaderException { Map<String, Object> templateMap = ConfigMain.transformTemplateToSchema(getClass().getClassLoader().getResourceAsStream(name + ".xml")).toMap(); Map<String, Object> yamlMap = SchemaLoader.loadYamlAsMap(getClass().getClassLoader().getResourceAsStream(name + ".yml")); http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/e6f6b066/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithFunnel.xml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithFunnel.xml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithFunnel.xml new file mode 100644 index 0000000..2e1b04f --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithFunnel.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + --> +<template><description></description><name>TemplateWithFunnel</name><snippet><funnels><id>e2c5b2f0-6740-4d57-a40b-20b27da784a3</id><parentGroupId>5e4f1cc2-78e9-4e08-a0c2-11779c9efc17</parentGroupId><position><x>2268.1912159884546</x><y>695.3930796314235</y></position></funnels></snippet><timestamp>09/21/2016 19:08:15 EDT</timestamp></template> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/e6f6b066/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithInputPort.xml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithInputPort.xml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithInputPort.xml new file mode 100644 index 0000000..37c2664 --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithInputPort.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + --> +<template><description></description><name>TemplateWithInputPort</name><snippet><inputPorts><id>8d2c579e-4ad2-4922-a311-a37b7e551b7a</id><parentGroupId>5e4f1cc2-78e9-4e08-a0c2-11779c9efc17</parentGroupId><position><x>2100.7989688172092</x><y>452.8720286116666</y></position><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><name>input port</name><state>STOPPED</state><transmitting>false</transmitting><type>INPUT_PORT</type><validationErrors>'Port 'input port'' is invalid because Output connection for port 'input port' is not defined.</validationErrors></inputPorts></snippet><timestamp>09/21/2016 19:05:44 EDT</timestamp></template> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/e6f6b066/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithOutputPort.xml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithOutputPort.xml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithOutputPort.xml new file mode 100644 index 0000000..2ceeddd --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithOutputPort.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + --> +<template><description></description><name>TemplateWithOutputPort</name><snippet><outputPorts><id>63421da6-2fd2-4732-8c5c-9782c1227fce</id><parentGroupId>5e4f1cc2-78e9-4e08-a0c2-11779c9efc17</parentGroupId><position><x>2547.617543425921</x><y>443.6456906809299</y></position><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><name>output port</name><state>STOPPED</state><transmitting>false</transmitting><type>OUTPUT_PORT</type></outputPorts></snippet><timestamp>09/21/2016 19:05:55 EDT</timestamp></template> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/e6f6b066/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithProcessGroup.xml ---------------------------------------------------------------------- diff --git a/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithProcessGroup.xml b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithProcessGroup.xml new file mode 100644 index 0000000..997e3ff --- /dev/null +++ b/minifi-toolkit/minifi-toolkit-configuration/src/test/resources/TemplateWithProcessGroup.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?><!-- +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + --> +<template><description></description><name>Template with Process Group</name><snippet><processGroups><id>95054577-00bf-4d2a-8484-35650c4ca28c</id><parentGroupId>5e4f1cc2-78e9-4e08-a0c2-11779c9efc17</parentGroupId><position><x>1468.1353574613213</x><y>611.0379314506386</y></position><activeRemotePortCount>0</activeRemotePortCount><comments></comments><contents><connections><id>fb54bf43-97e1-4e26-bf9a-33a9b3aba882</id><parentGroupId>95054577-00bf-4d2a-8484-35650c4ca28c</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>95054577-00bf-4d2a-8484-35650c4ca28c</groupId><id>810cb71c-dffc-4753-840f-44bcc9d6dcee</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>success</selectedRelationships><source><groupId>95054577-00bf-4d2a-8484-35650c4ca28c</groupId><id>403e2fcf-8f73-4955-8f08-4c6b 1c4d14d2</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><processors><id>810cb71c-dffc-4753-840f-44bcc9d6dcee</id><parentGroupId>95054577-00bf-4d2a-8484-35650c4ca28c</parentGroupId><position><x>494.7638029404583</x><y>790.1879571672059</y></position><config><annotationData><criteria> + <flowFilePolicy>USE_ORIGINAL</flowFilePolicy> +</criteria></annotationData><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Delete Attributes Expression</key><value><description>Regular expression for attributes to be deleted from flowfiles.</description><displayName>Delete Attributes Expression</displayName><dynamic>false</dynamic><name>Delete Attributes Expression</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>number</key><value><description></description><displayName>number</d isplayName><dynamic>true</dynamic><name>number</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Delete Attributes Expression</key></entry><entry><key>number</key><value>${nextInt():mod(10)}</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>UpdateAttribute</name><relationships><autoTerminate>true</autoTerminate><description>All FlowFiles are routed to this relationship</description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.attributes.UpdateAttribute</type></processors><processors><id>403e2fcf-8f 73-4955-8f08-4c6b1c4d14d2</id><parentGroupId>95054577-00bf-4d2a-8484-35650c4ca28c</parentGroupId><position><x>517.5545135105037</x><y>548.2988237337736</y></position><config><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>File Size</key><value><description>The size of the file that will be used</description><displayName>File Size</displayName><dynamic>false</dynamic><name>File Size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Batch Size </key><value><defaultValue>1</defaultValue><description>The number of FlowFiles to be transferred in each invocation</description><displayName>Batch Size</displayName><dynamic>false</dynamic><name>Batch Size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Data Format</key><value><allowableValues><displayName>Binary</displayName><value>Binary</value></allowableValues><allowableValues><displayName>Text</displayName><value>Text</value></allowableValues><defaultValue>Binary</defaultValue><description>Specifies whether the data should be Text or Binary</description><displayName>Data Format</displayName><dynamic>false</dynamic><name>Data Format</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Unique FlowFiles</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName>< value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, each FlowFile that is generated will be unique. If false, a random value will be generated and all FlowFiles will get the same content but this offers much higher throughput</description><displayName>Unique FlowFiles</displayName><dynamic>false</dynamic><name>Unique FlowFiles</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>File Size</key><value>10B</value></entry><entry><key>Batch Size</key><value>1</value></entry><entry><key>Data Format</key><value>Binary</value></entry><entry><key>Unique FlowFiles</key><value>false</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0/10 * * * * ?</schedulingPeriod><schedulingStrategy>CRON_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></co nfig><name>GenerateFlowFile</name><relationships><autoTerminate>false</autoTerminate><description></description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>false</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.GenerateFlowFile</type></processors></contents><disabledCount>0</disabledCount><inactiveRemotePortCount>0</inactiveRemotePortCount><inputPortCount>0</inputPortCount><invalidCount>0</invalidCount><name>test</name><outputPortCount>0</outputPortCount><parent><id>5e4f1cc2-78e9-4e08-a0c2-11779c9efc17</id><name>NiFi Flow</name></parent><runningCount>0</runningCount><stoppedCount>2</stoppedCount></processGroups></snippet><timestamp>09/21/2016 18:58:10 EDT</timestamp></template> \ No newline at end of file
