Repository: nifi-minifi Updated Branches: refs/heads/master f89f41504 -> 25298e90b
http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/MultipleRelationships.xml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/MultipleRelationships.xml b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/MultipleRelationships.xml new file mode 100644 index 0000000..af2dd1c --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/MultipleRelationships.xml @@ -0,0 +1,340 @@ +<?xml version="1.0" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<template encoding-version="1.0"> + <description></description> + <groupId>7c73bce2-0157-1000-1ea7-07d5d9c57c74</groupId> + <name>MultipleRelationships</name> + <snippet> + <connections> + <id>7c798ca3-0157-1000-0000-000000000000</id> + <parentGroupId>7c73bce2-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>7c73bce2-0157-1000-0000-000000000000</groupId> + <id>7c75ab71-0157-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>abc</selectedRelationships> + <selectedRelationships>unmatched</selectedRelationships> + <source> + <groupId>7c73bce2-0157-1000-0000-000000000000</groupId> + <id>7c768622-0157-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>7c79cce3-0157-1000-0000-000000000000</id> + <parentGroupId>7c73bce2-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>7c73bce2-0157-1000-0000-000000000000</groupId> + <id>7c79ba25-0157-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>7c73bce2-0157-1000-0000-000000000000</groupId> + <id>7c755ed6-0157-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>7c79d576-0157-1000-0000-000000000000</id> + <parentGroupId>7c73bce2-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>7c73bce2-0157-1000-0000-000000000000</groupId> + <id>7c768622-0157-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>7c73bce2-0157-1000-0000-000000000000</groupId> + <id>7c79ba25-0157-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <processors> + <id>7c755ed6-0157-1000-0000-000000000000</id> + <parentGroupId>7c73bce2-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>0.0</x> + <y>0.0</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>File Size</key> + <value> + <name>File Size</name> + </value> + </entry> + <entry> + <key>Batch Size</key> + <value> + <name>Batch Size</name> + </value> + </entry> + <entry> + <key>Data Format</key> + <value> + <name>Data Format</name> + </value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value> + <name>Unique FlowFiles</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>File Size</key> + <value>0 B</value> + </entry> + <entry> + <key>Batch Size</key> + <value>1</value> + </entry> + <entry> + <key>Data Format</key> + <value>Binary</value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value>false</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>GenerateFlowFile</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.GenerateFlowFile</type> + </processors> + <processors> + <id>7c75ab71-0157-1000-0000-000000000000</id> + <parentGroupId>7c73bce2-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>3.10443115234375</x> + <y>580.1210327148438</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>Log Level</key> + <value> + <name>Log Level</name> + </value> + </entry> + <entry> + <key>Log Payload</key> + <value> + <name>Log Payload</name> + </value> + </entry> + <entry> + <key>Attributes to Log</key> + <value> + <name>Attributes to Log</name> + </value> + </entry> + <entry> + <key>Attributes to Ignore</key> + <value> + <name>Attributes to Ignore</name> + </value> + </entry> + <entry> + <key>Log prefix</key> + <value> + <name>Log prefix</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Log Level</key> + <value>info</value> + </entry> + <entry> + <key>Log Payload</key> + <value>false</value> + </entry> + <entry> + <key>Attributes to Log</key> + </entry> + <entry> + <key>Attributes to Ignore</key> + </entry> + <entry> + <key>Log prefix</key> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>LogAttribute</name> + <relationships> + <autoTerminate>true</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.LogAttribute</type> + </processors> + <processors> + <id>7c768622-0157-1000-0000-000000000000</id> + <parentGroupId>7c73bce2-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>3.89630126953125</x> + <y>386.68919372558594</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>Routing Strategy</key> + <value> + <name>Routing Strategy</name> + </value> + </entry> + <entry> + <key>abc</key> + <value> + <name>abc</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Routing Strategy</key> + <value>Route to Property name</value> + </entry> + <entry> + <key>abc</key> + <value>${filename:equals('abc')}</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>RouteOnAttribute</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>abc</name> + </relationships> + <relationships> + <autoTerminate>false</autoTerminate> + <name>unmatched</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.RouteOnAttribute</type> + </processors> + <processors> + <id>7c79ba25-0157-1000-0000-000000000000</id> + <parentGroupId>7c73bce2-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>15.0</x> + <y>199.0</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>Delete Attributes Expression</key> + <value> + <name>Delete Attributes Expression</name> + </value> + </entry> + <entry> + <key>filename</key> + <value> + <name>filename</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Delete Attributes Expression</key> + </entry> + <entry> + <key>filename</key> + <value>abc</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>UpdateAttribute</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.attributes.UpdateAttribute</type> + </processors> + </snippet> + <timestamp>09/30/2016 15:03:09 EDT</timestamp> +</template> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/expected.json ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/expected.json new file mode 100644 index 0000000..6110f19 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/xml/expected.json @@ -0,0 +1,4 @@ +{ + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[.*\\] for LogAttribute\\[id=7c75ab71-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 20 +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/MultipleRelationships.yml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/MultipleRelationships.yml b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/MultipleRelationships.yml new file mode 100644 index 0000000..71dce07 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/MultipleRelationships.yml @@ -0,0 +1,153 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 2 +Flow Controller: + name: MultipleRelationships + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: 7c755ed6-0157-1000-0000-000000000000 + name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '1' + Data Format: Binary + File Size: 0 B + Unique FlowFiles: 'false' +- id: 7c75ab71-0157-1000-0000-000000000000 + name: LogAttribute + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: +- id: 7c768622-0157-1000-0000-000000000000 + name: RouteOnAttribute + class: org.apache.nifi.processors.standard.RouteOnAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Routing Strategy: Route to Property name + abc: ${filename:equals('abc')} +- id: 7c79ba25-0157-1000-0000-000000000000 + name: UpdateAttribute + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + filename: abc +Process Groups: [] +Input Ports: [] +Output Ports: [] +Funnels: [] +Connections: +- id: 7c79cce3-0157-1000-0000-000000000000 + name: GenerateFlowFile/success/UpdateAttribute + source id: 7c755ed6-0157-1000-0000-000000000000 + source relationship names: + - success + destination id: 7c79ba25-0157-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 7c798ca3-0157-1000-0000-000000000000 + name: RouteOnAttribute/abc/LogAttribute + source id: 7c768622-0157-1000-0000-000000000000 + source relationship names: + - abc + - unmatched + destination id: 7c75ab71-0157-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 7c79d576-0157-1000-0000-000000000000 + name: UpdateAttribute/success/RouteOnAttribute + source id: 7c79ba25-0157-1000-0000-000000000000 + source relationship names: + - success + destination id: 7c768622-0157-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +Remote Process Groups: [] http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json new file mode 100644 index 0000000..6110f19 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/MultipleRelationships/yml/expected.json @@ -0,0 +1,4 @@ +{ + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[.*\\] for LogAttribute\\[id=7c75ab71-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 20 +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/ProcessGroups.xml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/ProcessGroups.xml b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/ProcessGroups.xml new file mode 100644 index 0000000..c5d5812 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/ProcessGroups.xml @@ -0,0 +1,370 @@ +<?xml version="1.0" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<template encoding-version="1.0"> + <description></description> + <groupId>e25c3a84-0157-1000-b0cb-8ebdb45e8945</groupId> + <name>pgtest</name> + <snippet> + <connections> + <id>e25dbff3-0157-1000-0000-000000000000</id> + <parentGroupId>e25c3a84-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>e25cbc20-0157-1000-0000-000000000000</groupId> + <id>e25d0cef-0157-1000-0000-000000000000</id> + <type>INPUT_PORT</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>e25c3a84-0157-1000-0000-000000000000</groupId> + <id>e25cd92a-0157-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>e25e22b1-0157-1000-0000-000000000000</id> + <parentGroupId>e25c3a84-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>e25c3a84-0157-1000-0000-000000000000</groupId> + <id>e25e0e6e-0157-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <source> + <groupId>e25cbc20-0157-1000-0000-000000000000</groupId> + <id>e25d578c-0157-1000-0000-000000000000</id> + <type>OUTPUT_PORT</type> + </source> + <zIndex>0</zIndex> + </connections> + <processGroups> + <id>e25cbc20-0157-1000-0000-000000000000</id> + <parentGroupId>e25c3a84-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>0.0</x> + <y>230.999990940094</y> + </position> + <comments></comments> + <contents> + <connections> + <id>e25d35dc-0157-1000-0000-000000000000</id> + <parentGroupId>e25cbc20-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>e25cbc20-0157-1000-0000-000000000000</groupId> + <id>e25d2588-0157-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <source> + <groupId>e25cbc20-0157-1000-0000-000000000000</groupId> + <id>e25d0cef-0157-1000-0000-000000000000</id> + <type>INPUT_PORT</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>e25d6bd6-0157-1000-0000-000000000000</id> + <parentGroupId>e25cbc20-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>e25cbc20-0157-1000-0000-000000000000</groupId> + <id>e25d578c-0157-1000-0000-000000000000</id> + <type>OUTPUT_PORT</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>e25cbc20-0157-1000-0000-000000000000</groupId> + <id>e25d2588-0157-1000-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <inputPorts> + <id>e25d0cef-0157-1000-0000-000000000000</id> + <parentGroupId>e25cbc20-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>457.0</x> + <y>50.0</y> + </position> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <name>input</name> + <state>STOPPED</state> + <type>INPUT_PORT</type> + </inputPorts> + <outputPorts> + <id>e25d578c-0157-1000-0000-000000000000</id> + <parentGroupId>e25cbc20-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>461.0</x> + <y>478.0</y> + </position> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <name>output</name> + <state>STOPPED</state> + <type>OUTPUT_PORT</type> + </outputPorts> + <processors> + <id>e25d2588-0157-1000-0000-000000000000</id> + <parentGroupId>e25cbc20-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>406.0</x> + <y>225.0</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>Log Level</key> + <value> + <name>Log Level</name> + </value> + </entry> + <entry> + <key>Log Payload</key> + <value> + <name>Log Payload</name> + </value> + </entry> + <entry> + <key>Attributes to Log</key> + <value> + <name>Attributes to Log</name> + </value> + </entry> + <entry> + <key>Attributes to Ignore</key> + <value> + <name>Attributes to Ignore</name> + </value> + </entry> + <entry> + <key>Log prefix</key> + <value> + <name>Log prefix</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Log Level</key> + <value>info</value> + </entry> + <entry> + <key>Log Payload</key> + <value>false</value> + </entry> + <entry> + <key>Attributes to Log</key> + </entry> + <entry> + <key>Attributes to Ignore</key> + </entry> + <entry> + <key>Log prefix</key> + <value>INSIDE</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>LogAttribute</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.LogAttribute</type> + </processors> + </contents> + <name>black box</name> + </processGroups> + <processors> + <id>e25cd92a-0157-1000-0000-000000000000</id> + <parentGroupId>e25c3a84-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>12.0</x> + <y>0.0</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>File Size</key> + <value> + <name>File Size</name> + </value> + </entry> + <entry> + <key>Batch Size</key> + <value> + <name>Batch Size</name> + </value> + </entry> + <entry> + <key>Data Format</key> + <value> + <name>Data Format</name> + </value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value> + <name>Unique FlowFiles</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>File Size</key> + <value>1 b</value> + </entry> + <entry> + <key>Batch Size</key> + <value>1</value> + </entry> + <entry> + <key>Data Format</key> + <value>Binary</value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value>false</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>GenerateFlowFile</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.GenerateFlowFile</type> + </processors> + <processors> + <id>e25e0e6e-0157-1000-0000-000000000000</id> + <parentGroupId>e25c3a84-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>21.08323973697111</x> + <y>461.2776537085565</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>Log Level</key> + <value> + <name>Log Level</name> + </value> + </entry> + <entry> + <key>Log Payload</key> + <value> + <name>Log Payload</name> + </value> + </entry> + <entry> + <key>Attributes to Log</key> + <value> + <name>Attributes to Log</name> + </value> + </entry> + <entry> + <key>Attributes to Ignore</key> + <value> + <name>Attributes to Ignore</name> + </value> + </entry> + <entry> + <key>Log prefix</key> + <value> + <name>Log prefix</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Log Level</key> + <value>info</value> + </entry> + <entry> + <key>Log Payload</key> + <value>false</value> + </entry> + <entry> + <key>Attributes to Log</key> + </entry> + <entry> + <key>Attributes to Ignore</key> + </entry> + <entry> + <key>Log prefix</key> + <value>OUTSIDE</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>LogAttribute</name> + <relationships> + <autoTerminate>true</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.LogAttribute</type> + </processors> + </snippet> + <timestamp>10/20/2016 09:53:44 EDT</timestamp> +</template> http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/expected.json ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/expected.json new file mode 100644 index 0000000..a1e4e43 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/xml/expected.json @@ -0,0 +1,4 @@ +{ + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=e25e0e6e-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 20 +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/ProcessGroups.yml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/ProcessGroups.yml b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/ProcessGroups.yml new file mode 100644 index 0000000..fc837b6 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/ProcessGroups.yml @@ -0,0 +1,163 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 2 +Flow Controller: + name: pgtest + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: e25cd92a-0157-1000-0000-000000000000 + name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '1' + Data Format: Binary + File Size: 1 b + Unique FlowFiles: 'false' +- id: e25e0e6e-0157-1000-0000-000000000000 + name: LogAttribute + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: + - success + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: OUTSIDE +Process Groups: +- id: e25cbc20-0157-1000-0000-000000000000 + name: black box + Processors: + - id: e25d2588-0157-1000-0000-000000000000 + name: LogAttribute + class: org.apache.nifi.processors.standard.LogAttribute + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Attributes to Ignore: + Attributes to Log: + Log Level: info + Log Payload: 'false' + Log prefix: INSIDE + Process Groups: [] + Input Ports: + - id: e25d0cef-0157-1000-0000-000000000000 + name: input + Output Ports: + - id: e25d578c-0157-1000-0000-000000000000 + name: output + Funnels: [] + Connections: + - id: e25d6bd6-0157-1000-0000-000000000000 + name: LogAttribute/success/output + source id: e25d2588-0157-1000-0000-000000000000 + source relationship names: + - success + destination id: e25d578c-0157-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' + - id: e25d35dc-0157-1000-0000-000000000000 + name: input//LogAttribute + source id: e25d0cef-0157-1000-0000-000000000000 + source relationship names: [] + destination id: e25d2588-0157-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' + Remote Process Groups: [] +Input Ports: [] +Output Ports: [] +Funnels: [] +Connections: +- id: e25dbff3-0157-1000-0000-000000000000 + name: GenerateFlowFile/success/input + source id: e25cd92a-0157-1000-0000-000000000000 + source relationship names: + - success + destination id: e25d0cef-0157-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: e25e22b1-0157-1000-0000-000000000000 + name: output//LogAttribute + source id: e25d578c-0157-1000-0000-000000000000 + source relationship names: [] + destination id: e25e0e6e-0157-1000-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +Remote Process Groups: [] http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json new file mode 100644 index 0000000..a1e4e43 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/ProcessGroups/yml/expected.json @@ -0,0 +1,4 @@ +{ + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for LogAttribute\\[id=e25e0e6e-0157-1000-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'success'", + "occurrences": 20 +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/StressTestFramework.xml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/StressTestFramework.xml b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/StressTestFramework.xml new file mode 100644 index 0000000..4234170 --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/StressTestFramework.xml @@ -0,0 +1,539 @@ +<?xml version="1.0" ?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + +<template encoding-version="1.0"> + <description></description> + <groupId>7cba7223-0157-1000-0571-63471fdb84c9</groupId> + <name>StressTestFramework</name> + <snippet> + <connections> + <id>4ad21393-16a4-1794-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>4ad21392-16a4-1794-0000-000000000000</id> + <type>FUNNEL</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>16a47794-5391-4ad2-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>4ad21394-16a4-1794-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>4ad21392-16a4-1794-0000-000000000000</id> + <type>FUNNEL</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>53914ad2-7794-16a4-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>4ad21395-16a4-1794-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>4ad21392-16a4-1794-0000-000000000000</id> + <type>FUNNEL</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>779416a4-4ad2-1391-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>4ad21396-16a4-1794-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>4ad21392-16a4-1794-0000-000000000000</id> + <type>FUNNEL</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>4ad21391-16a4-1794-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>4ad21397-16a4-1794-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>92557c76-f251-45a4-0000-000000000000</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <source> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>4ad21392-16a4-1794-0000-000000000000</id> + <type>FUNNEL</type> + </source> + <zIndex>0</zIndex> + </connections> + <connections> + <id>4c53556e-eb46-458c-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>2000</backPressureObjectThreshold> + <destination> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>397a4910-cc01-4c6b-0000-000000000000</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <prioritizers>org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer</prioritizers> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>7cba7223-0157-1000-0000-000000000000</groupId> + <id>92557c76-f251-45a4-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <funnels> + <id>4ad21392-16a4-1794-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>593.8762672567133</x> + <y>246.40484683047555</y> + </position> + </funnels> + <processors> + <id>92557c76-f251-45a4-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>442.8776092529297</x> + <y>345.48790588378904</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>2</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>Delete Attributes Expression</key> + <value> + <name>Delete Attributes Expression</name> + </value> + </entry> + <entry> + <key>property 1</key> + <value> + <name>property 1</name> + </value> + </entry> + <entry> + <key>property 2</key> + <value> + <name>property 2</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Delete Attributes Expression</key> + </entry> + <entry> + <key>property 1</key> + <value>value 1</value> + </entry> + <entry> + <key>property 2</key> + <value>value 2 ${nextInt()}</value> + </entry> + </properties> + <runDurationMillis>100</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>UpdateAttribute</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.attributes.UpdateAttribute</type> + </processors> + <processors> + <id>16a47794-5391-4ad2-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>0.0</x> + <y>188.24237060546875</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>File Size</key> + <value> + <name>File Size</name> + </value> + </entry> + <entry> + <key>Batch Size</key> + <value> + <name>Batch Size</name> + </value> + </entry> + <entry> + <key>Data Format</key> + <value> + <name>Data Format</name> + </value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value> + <name>Unique FlowFiles</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>File Size</key> + <value>0 KB</value> + </entry> + <entry> + <key>Batch Size</key> + <value>100</value> + </entry> + <entry> + <key>Data Format</key> + <value>Binary</value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value>false</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>GenerateFlowFile</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.GenerateFlowFile</type> + </processors> + <processors> + <id>397a4910-cc01-4c6b-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>444.3776092529297</x> + <y>609.6672564697267</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>2</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>Routing Strategy</key> + <value> + <name>Routing Strategy</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Routing Strategy</key> + <value>Route to Property name</value> + </entry> + </properties> + <runDurationMillis>50</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>RouteOnAttribute</name> + <relationships> + <autoTerminate>true</autoTerminate> + <name>unmatched</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.RouteOnAttribute</type> + </processors> + <processors> + <id>4ad21391-16a4-1794-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>975.3996429443359</x> + <y>173.76165771484375</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>File Size</key> + <value> + <name>File Size</name> + </value> + </entry> + <entry> + <key>Batch Size</key> + <value> + <name>Batch Size</name> + </value> + </entry> + <entry> + <key>Data Format</key> + <value> + <name>Data Format</name> + </value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value> + <name>Unique FlowFiles</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>File Size</key> + <value>0 KB</value> + </entry> + <entry> + <key>Batch Size</key> + <value>100</value> + </entry> + <entry> + <key>Data Format</key> + <value>Binary</value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value>false</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>GenerateFlowFile</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.GenerateFlowFile</type> + </processors> + <processors> + <id>53914ad2-7794-16a4-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>174.88023376464844</x> + <y>9.67376708984375</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>File Size</key> + <value> + <name>File Size</name> + </value> + </entry> + <entry> + <key>Batch Size</key> + <value> + <name>Batch Size</name> + </value> + </entry> + <entry> + <key>Data Format</key> + <value> + <name>Data Format</name> + </value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value> + <name>Unique FlowFiles</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>File Size</key> + <value>0 KB</value> + </entry> + <entry> + <key>Batch Size</key> + <value>100</value> + </entry> + <entry> + <key>Data Format</key> + <value>Binary</value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value>false</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>GenerateFlowFile</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.GenerateFlowFile</type> + </processors> + <processors> + <id>779416a4-4ad2-1391-0000-000000000000</id> + <parentGroupId>7cba7223-0157-1000-0000-000000000000</parentGroupId> + <position> + <x>758.0161590576172</x> + <y>0.0</y> + </position> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>File Size</key> + <value> + <name>File Size</name> + </value> + </entry> + <entry> + <key>Batch Size</key> + <value> + <name>Batch Size</name> + </value> + </entry> + <entry> + <key>Data Format</key> + <value> + <name>Data Format</name> + </value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value> + <name>Unique FlowFiles</name> + </value> + </entry> + </descriptors> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>File Size</key> + <value>0 KB</value> + </entry> + <entry> + <key>Batch Size</key> + <value>100</value> + </entry> + <entry> + <key>Data Format</key> + <value>Binary</value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value>false</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>GenerateFlowFile</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <style></style> + <type>org.apache.nifi.processors.standard.GenerateFlowFile</type> + </processors> + </snippet> + <timestamp>09/30/2016 16:18:24 EDT</timestamp> +</template> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/expected.json ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/expected.json new file mode 100644 index 0000000..5f3740b --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/xml/expected.json @@ -0,0 +1,4 @@ +{ + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for RouteOnAttribute\\[id=397a4910-cc01-4c6b-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'unmatched'", + "occurrences": 20 +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/StressTestFramework.yml ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/StressTestFramework.yml b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/StressTestFramework.yml new file mode 100644 index 0000000..46cd84d --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/StressTestFramework.yml @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +MiNiFi Config Version: 2 +Flow Controller: + name: StressTestFramework + comment: '' +Core Properties: + flow controller graceful shutdown period: 10 sec + flow service write delay interval: 500 ms + administrative yield duration: 30 sec + bored yield duration: 10 millis + max concurrent threads: 1 +FlowFile Repository: + partitions: 256 + checkpoint interval: 2 mins + always sync: false + Swap: + threshold: 20000 + in period: 5 sec + in threads: 1 + out period: 5 sec + out threads: 4 +Content Repository: + content claim max appendable size: 10 MB + content claim max flow files: 100 + always sync: false +Provenance Repository: + provenance rollover time: 1 min +Component Status Repository: + buffer size: 1440 + snapshot frequency: 1 min +Security Properties: + keystore: '' + keystore type: '' + keystore password: '' + key password: '' + truststore: '' + truststore type: '' + truststore password: '' + ssl protocol: '' + Sensitive Props: + key: + algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL + provider: BC +Processors: +- id: 16a47794-5391-4ad2-0000-000000000000 + name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '100' + Data Format: Binary + File Size: 0 KB + Unique FlowFiles: 'false' +- id: 4ad21391-16a4-1794-0000-000000000000 + name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '100' + Data Format: Binary + File Size: 0 KB + Unique FlowFiles: 'false' +- id: 53914ad2-7794-16a4-0000-000000000000 + name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '100' + Data Format: Binary + File Size: 0 KB + Unique FlowFiles: 'false' +- id: 779416a4-4ad2-1391-0000-000000000000 + name: GenerateFlowFile + class: org.apache.nifi.processors.standard.GenerateFlowFile + max concurrent tasks: 1 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 0 + auto-terminated relationships list: [] + Properties: + Batch Size: '100' + Data Format: Binary + File Size: 0 KB + Unique FlowFiles: 'false' +- id: 397a4910-cc01-4c6b-0000-000000000000 + name: RouteOnAttribute + class: org.apache.nifi.processors.standard.RouteOnAttribute + max concurrent tasks: 2 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 50000 + auto-terminated relationships list: + - unmatched + Properties: + Routing Strategy: Route to Property name +- id: 92557c76-f251-45a4-0000-000000000000 + name: UpdateAttribute + class: org.apache.nifi.processors.attributes.UpdateAttribute + max concurrent tasks: 2 + scheduling strategy: TIMER_DRIVEN + scheduling period: 0 sec + penalization period: 30 sec + yield period: 1 sec + run duration nanos: 100000 + auto-terminated relationships list: [] + Properties: + Delete Attributes Expression: + property 1: value 1 + property 2: value 2 ${nextInt()} +Process Groups: [] +Input Ports: [] +Output Ports: [] +Funnels: +- id: 4ad21392-16a4-1794-0000-000000000000 +Connections: +- id: 4ad21397-16a4-1794-0000-000000000000 + name: 4ad21392-16a4-1794-0000-000000000000//UpdateAttribute + source id: 4ad21392-16a4-1794-0000-000000000000 + source relationship names: [] + destination id: 92557c76-f251-45a4-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 4ad21393-16a4-1794-0000-000000000000 + name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 + source id: 16a47794-5391-4ad2-0000-000000000000 + source relationship names: + - success + destination id: 4ad21392-16a4-1794-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 4ad21394-16a4-1794-0000-000000000000 + name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 + source id: 53914ad2-7794-16a4-0000-000000000000 + source relationship names: + - success + destination id: 4ad21392-16a4-1794-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 4ad21395-16a4-1794-0000-000000000000 + name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 + source id: 779416a4-4ad2-1391-0000-000000000000 + source relationship names: + - success + destination id: 4ad21392-16a4-1794-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 4ad21396-16a4-1794-0000-000000000000 + name: GenerateFlowFile/success/4ad21392-16a4-1794-0000-000000000000 + source id: 4ad21391-16a4-1794-0000-000000000000 + source relationship names: + - success + destination id: 4ad21392-16a4-1794-0000-000000000000 + max work queue size: 10000 + max work queue data size: 1 GB + flowfile expiration: 0 sec + queue prioritizer class: '' +- id: 4c53556e-eb46-458c-0000-000000000000 + name: UpdateAttribute/success/RouteOnAttribute + source id: 92557c76-f251-45a4-0000-000000000000 + source relationship names: + - success + destination id: 397a4910-cc01-4c6b-0000-000000000000 + max work queue size: 2000 + max work queue data size: 0 MB + flowfile expiration: 0 sec + queue prioritizer class: org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer +Remote Process Groups: [] http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json new file mode 100644 index 0000000..5f3740b --- /dev/null +++ b/minifi-integration-tests/src/test/resources/standalone/v2/StressTestFramework/yml/expected.json @@ -0,0 +1,4 @@ +{ + "pattern": "o.a.n.c.r.StandardProcessSession StandardProcessSession\\[id=.*\\] for RouteOnAttribute\\[id=397a4910-cc01-4c6b-0000-000000000000\\], committed the following events: Transferred FlowFiles \\[.*\\] to 'unmatched'", + "occurrences": 20 +} http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/minifi-integration-tests/src/test/resources/tailFileServer.py ---------------------------------------------------------------------- diff --git a/minifi-integration-tests/src/test/resources/tailFileServer.py b/minifi-integration-tests/src/test/resources/tailFileServer.py new file mode 100644 index 0000000..bec2ccf --- /dev/null +++ b/minifi-integration-tests/src/test/resources/tailFileServer.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the \"License\"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python + +import logging + +from argparse import ArgumentParser +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +from SocketServer import ThreadingMixIn +from subprocess import PIPE, Popen + +# Needs to be threaded or health check hangs the server +class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): + pass + +class TailHTTPRequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-type','text/plain') + self.end_headers() + p = Popen(['tail', '-f', '-n', '+1', TAIL_FILE], stdout=PIPE) + try: + for line in iter(p.stdout.readline, b''): + self.wfile.write(line) + self.wfile.flush() + finally: + p.kill() + return + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + parser = ArgumentParser(description='Tail file over http') + parser.add_argument('--file') + parser.add_argument('--port', type=int, default=8000) + + logging.debug('About to parse arguments') + args = parser.parse_args() + + if not args.file: + raise Exception('Must specify --file') + + global TAIL_FILE + TAIL_FILE = args.file + + logging.debug('Serving tail of ' + TAIL_FILE + ' via HTTP at port ' + str(args.port)) + + server = ThreadedHTTPServer(('', args.port), TailHTTPRequestHandler) + server.serve_forever() \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi-minifi/blob/25298e90/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3fa1f2a..7ea5a2c 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ limitations under the License. <module>minifi-toolkit</module> <module>minifi-docker</module> <module>minifi-c2</module> + <module>minifi-integration-tests</module> </modules> <url>http://nifi.apache.org/minifi</url>
