Hi all,
Case1:
While working with a single file or couple of files flow seems to be
working good.
Case2:
While working with large number of files with 17mb/queue and total of 5
queues (around 75mb)
file gets into queue but execute processor doesn't pick up the file and
hangs.
and unable to restart the processor once it is stopped.
empty queue will not empty all the queue but a single file is left off.
which can be cleared.
Need to restart NIFI and clear the queue.
I quite unsure if the problem is with configuring NIFI like flowfile
repository.
below attached is my xml template.
Thanks,
Raghu
<?xml version="1.0" encoding="UTF-8" standalone="yes"?><template><description></description><name>Dataingest_opentsdb</name><snippet><connections><id>66443fde-e68e-493e-a00b-50047aadb36d</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>97e78d73-79b0-4a98-8d76-27302adaf957</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>2</selectedRelationships><source><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>47a10ff4-ea4c-47cd-a2dc-367f62f67705</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>43f7f7da-1de8-4c2b-a003-8b6964867afa</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>47a10ff4-ea4c-47cd-a2dc-367f62f67705</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>failure</selectedRelationships><selectedRelationships>success</selectedRelationships><source><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>c0115804-cd40-4174-8bbf-ece12e89965d</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>b8c99218-ccae-4f2f-ae75-a33793142cf5</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>42db8d0e-a0d8-4863-8922-561187e0d8fc</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>4</selectedRelationships><source><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>47a10ff4-ea4c-47cd-a2dc-367f62f67705</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>5170174f-f6b6-42e0-b5c5-535e76dcdc85</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>de08d96a-02e6-4bab-ac6c-26125292d6c7</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>5</selectedRelationships><source><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>47a10ff4-ea4c-47cd-a2dc-367f62f67705</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>353e7e3b-a22e-4c9d-a183-40ba83f7ecb8</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>efa70804-6be3-4314-b741-205a2da51ba3</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>3</selectedRelationships><source><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>47a10ff4-ea4c-47cd-a2dc-367f62f67705</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>ef3e3974-16ba-470f-9040-3f71d283befa</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>c0115804-cd40-4174-8bbf-ece12e89965d</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>success</selectedRelationships><source><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>9da7293b-c0c3-4db1-a55f-ca9efdde58ab</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>eb3b240c-2ffa-4203-80df-af28a2c64329</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>6d751ee6-a4c9-4a26-8dde-e3ce001320db</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>1</selectedRelationships><source><groupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</groupId><id>47a10ff4-ea4c-47cd-a2dc-367f62f67705</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><labels><id>0a98f4a6-bea4-4931-a30b-414701a415fe</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><position><x>775.5489887142542</x><y>614.4553962161779</y></position><height>150.0</height><label>get file = /import/origin
python script = changeserialno.py</label><style><entry><key>font-size</key><value>12px</value></entry></style><width>150.0</width></labels><processors><id>de08d96a-02e6-4bab-ac6c-26125292d6c7</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><position><x>2230.1046077872534</x><y>1392.703884322121</y></position><config><bulletinLevel>DEBUG</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Command Arguments</key><value><description>The arguments to supply to the executable delimited by the ';' character.</description><displayName>Command Arguments</displayName><dynamic>false</dynamic><name>Command Arguments</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Command Path</key><value><description>Specifies the command to be executed; if just the name of an executable is provided, it must be in the user's environment PATH.</description><displayName>Command Path</displayName><dynamic>false</dynamic><name>Command Path</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Ignore STDIN</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, the contents of the incoming flowfile will not be passed to the executing command</description><displayName>Ignore STDIN</displayName><dynamic>false</dynamic><name>Ignore STDIN</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Working Directory</key><value><description>The directory to use as the current working directory when executing the command</description><displayName>Working Directory</displayName><dynamic>false</dynamic><name>Working Directory</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Argument Delimiter</key><value><defaultValue>;</defaultValue><description>Delimiter to use to separate arguments for a command [default: ;]. Must be a single character</description><displayName>Argument Delimiter</displayName><dynamic>false</dynamic><name>Argument Delimiter</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Output Destination Attribute</key><value><description>If set, the output of the stream command will be put into an attribute of the original FlowFile instead of a separate FlowFile. There will no longer be a relationship for 'output stream'. The value of this property will be the key for the output attribute.</description><displayName>Output Destination Attribute</displayName><dynamic>false</dynamic><name>Output Destination Attribute</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Max Attribute Length</key><value><defaultValue>256</defaultValue><description>If routing the output of the stream command to an attribute, the number of characters put to the attribute value will be at most this amount. This is important because attributes are held in memory and large attributes will quickly cause out of memory issues. If the output goes longer than this value, it will truncated to fit. Consider making this smaller if able.</description><displayName>Max Attribute Length</displayName><dynamic>false</dynamic><name>Max Attribute Length</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Command Arguments</key><value>${absolute.path};${filename};/import/TagListDataOnly.csv;/import/changed/;/import/final/;/import/intermediate/;</value></entry><entry><key>Command Path</key><value>/import/pythonscripts/ETL1.py</value></entry><entry><key>Ignore STDIN</key><value>false</value></entry><entry><key>Working Directory</key></entry><entry><key>Argument Delimiter</key><value>;</value></entry><entry><key>Output Destination Attribute</key><value>output stream</value></entry><entry><key>Max Attribute Length</key><value>256</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>ExecuteStreamCommand</name><relationships><autoTerminate>true</autoTerminate><description>FlowFiles that were successfully processed</description><name>original</name></relationships><state>STOPPED</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.ExecuteStreamCommand</type></processors><processors><id>42db8d0e-a0d8-4863-8922-561187e0d8fc</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><position><x>2226.22334738273</x><y>1237.4562033534523</y></position><config><bulletinLevel>DEBUG</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Command Arguments</key><value><description>The arguments to supply to the executable delimited by the ';' character.</description><displayName>Command Arguments</displayName><dynamic>false</dynamic><name>Command Arguments</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Command Path</key><value><description>Specifies the command to be executed; if just the name of an executable is provided, it must be in the user's environment PATH.</description><displayName>Command Path</displayName><dynamic>false</dynamic><name>Command Path</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Ignore STDIN</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, the contents of the incoming flowfile will not be passed to the executing command</description><displayName>Ignore STDIN</displayName><dynamic>false</dynamic><name>Ignore STDIN</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Working Directory</key><value><description>The directory to use as the current working directory when executing the command</description><displayName>Working Directory</displayName><dynamic>false</dynamic><name>Working Directory</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Argument Delimiter</key><value><defaultValue>;</defaultValue><description>Delimiter to use to separate arguments for a command [default: ;]. Must be a single character</description><displayName>Argument Delimiter</displayName><dynamic>false</dynamic><name>Argument Delimiter</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Output Destination Attribute</key><value><description>If set, the output of the stream command will be put into an attribute of the original FlowFile instead of a separate FlowFile. There will no longer be a relationship for 'output stream'. The value of this property will be the key for the output attribute.</description><displayName>Output Destination Attribute</displayName><dynamic>false</dynamic><name>Output Destination Attribute</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Max Attribute Length</key><value><defaultValue>256</defaultValue><description>If routing the output of the stream command to an attribute, the number of characters put to the attribute value will be at most this amount. This is important because attributes are held in memory and large attributes will quickly cause out of memory issues. If the output goes longer than this value, it will truncated to fit. Consider making this smaller if able.</description><displayName>Max Attribute Length</displayName><dynamic>false</dynamic><name>Max Attribute Length</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Command Arguments</key><value>${absolute.path};${filename};/import/TagListDataOnly.csv;/import/changed/;/import/final/;/import/intermediate/;</value></entry><entry><key>Command Path</key><value>/import/pythonscripts/ETL1.py</value></entry><entry><key>Ignore STDIN</key><value>false</value></entry><entry><key>Working Directory</key></entry><entry><key>Argument Delimiter</key><value>;</value></entry><entry><key>Output Destination Attribute</key><value>output stream</value></entry><entry><key>Max Attribute Length</key><value>256</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>ExecuteStreamCommand</name><relationships><autoTerminate>true</autoTerminate><description>FlowFiles that were successfully processed</description><name>original</name></relationships><state>STOPPED</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.ExecuteStreamCommand</type></processors><processors><id>efa70804-6be3-4314-b741-205a2da51ba3</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><position><x>2218.4613785190513</x><y>1064.0960715360684</y></position><config><bulletinLevel>DEBUG</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Command Arguments</key><value><description>The arguments to supply to the executable delimited by the ';' character.</description><displayName>Command Arguments</displayName><dynamic>false</dynamic><name>Command Arguments</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Command Path</key><value><description>Specifies the command to be executed; if just the name of an executable is provided, it must be in the user's environment PATH.</description><displayName>Command Path</displayName><dynamic>false</dynamic><name>Command Path</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Ignore STDIN</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, the contents of the incoming flowfile will not be passed to the executing command</description><displayName>Ignore STDIN</displayName><dynamic>false</dynamic><name>Ignore STDIN</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Working Directory</key><value><description>The directory to use as the current working directory when executing the command</description><displayName>Working Directory</displayName><dynamic>false</dynamic><name>Working Directory</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Argument Delimiter</key><value><defaultValue>;</defaultValue><description>Delimiter to use to separate arguments for a command [default: ;]. Must be a single character</description><displayName>Argument Delimiter</displayName><dynamic>false</dynamic><name>Argument Delimiter</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Output Destination Attribute</key><value><description>If set, the output of the stream command will be put into an attribute of the original FlowFile instead of a separate FlowFile. There will no longer be a relationship for 'output stream'. The value of this property will be the key for the output attribute.</description><displayName>Output Destination Attribute</displayName><dynamic>false</dynamic><name>Output Destination Attribute</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Max Attribute Length</key><value><defaultValue>256</defaultValue><description>If routing the output of the stream command to an attribute, the number of characters put to the attribute value will be at most this amount. This is important because attributes are held in memory and large attributes will quickly cause out of memory issues. If the output goes longer than this value, it will truncated to fit. Consider making this smaller if able.</description><displayName>Max Attribute Length</displayName><dynamic>false</dynamic><name>Max Attribute Length</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Command Arguments</key><value>${absolute.path};${filename};/import/TagListDataOnly.csv;/import/changed/;/import/final/;/import/intermediate/;</value></entry><entry><key>Command Path</key><value>/import/pythonscripts/ETL1.py</value></entry><entry><key>Ignore STDIN</key><value>false</value></entry><entry><key>Working Directory</key></entry><entry><key>Argument Delimiter</key><value>;</value></entry><entry><key>Output Destination Attribute</key><value>output stream</value></entry><entry><key>Max Attribute Length</key><value>256</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>ExecuteStreamCommand</name><relationships><autoTerminate>true</autoTerminate><description>FlowFiles that were successfully processed</description><name>original</name></relationships><state>STOPPED</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.ExecuteStreamCommand</type></processors><processors><id>6d751ee6-a4c9-4a26-8dde-e3ce001320db</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><position><x>2209.6916826489305</x><y>736.7680229759216</y></position><config><bulletinLevel>DEBUG</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Command Arguments</key><value><description>The arguments to supply to the executable delimited by the ';' character.</description><displayName>Command Arguments</displayName><dynamic>false</dynamic><name>Command Arguments</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Command Path</key><value><description>Specifies the command to be executed; if just the name of an executable is provided, it must be in the user's environment PATH.</description><displayName>Command Path</displayName><dynamic>false</dynamic><name>Command Path</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Ignore STDIN</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, the contents of the incoming flowfile will not be passed to the executing command</description><displayName>Ignore STDIN</displayName><dynamic>false</dynamic><name>Ignore STDIN</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Working Directory</key><value><description>The directory to use as the current working directory when executing the command</description><displayName>Working Directory</displayName><dynamic>false</dynamic><name>Working Directory</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Argument Delimiter</key><value><defaultValue>;</defaultValue><description>Delimiter to use to separate arguments for a command [default: ;]. Must be a single character</description><displayName>Argument Delimiter</displayName><dynamic>false</dynamic><name>Argument Delimiter</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Output Destination Attribute</key><value><description>If set, the output of the stream command will be put into an attribute of the original FlowFile instead of a separate FlowFile. There will no longer be a relationship for 'output stream'. The value of this property will be the key for the output attribute.</description><displayName>Output Destination Attribute</displayName><dynamic>false</dynamic><name>Output Destination Attribute</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Max Attribute Length</key><value><defaultValue>256</defaultValue><description>If routing the output of the stream command to an attribute, the number of characters put to the attribute value will be at most this amount. This is important because attributes are held in memory and large attributes will quickly cause out of memory issues. If the output goes longer than this value, it will truncated to fit. Consider making this smaller if able.</description><displayName>Max Attribute Length</displayName><dynamic>false</dynamic><name>Max Attribute Length</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Command Arguments</key><value>${absolute.path};${filename};/import/TagListDataOnly.csv;/import/changed/;/import/final/;/import/intermediate/;</value></entry><entry><key>Command Path</key><value>/import/pythonscripts/ETL1.py</value></entry><entry><key>Ignore STDIN</key><value>false</value></entry><entry><key>Working Directory</key></entry><entry><key>Argument Delimiter</key><value>;</value></entry><entry><key>Output Destination Attribute</key><value>output stream</value></entry><entry><key>Max Attribute Length</key><value>256</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>ExecuteStreamCommand</name><relationships><autoTerminate>true</autoTerminate><description>FlowFiles that were successfully processed</description><name>original</name></relationships><state>STOPPED</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.ExecuteStreamCommand</type></processors><processors><id>97e78d73-79b0-4a98-8d76-27302adaf957</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><position><x>2213.286360562782</x><y>911.4355419895714</y></position><config><bulletinLevel>DEBUG</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Command Arguments</key><value><description>The arguments to supply to the executable delimited by the ';' character.</description><displayName>Command Arguments</displayName><dynamic>false</dynamic><name>Command Arguments</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Command Path</key><value><description>Specifies the command to be executed; if just the name of an executable is provided, it must be in the user's environment PATH.</description><displayName>Command Path</displayName><dynamic>false</dynamic><name>Command Path</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Ignore STDIN</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, the contents of the incoming flowfile will not be passed to the executing command</description><displayName>Ignore STDIN</displayName><dynamic>false</dynamic><name>Ignore STDIN</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Working Directory</key><value><description>The directory to use as the current working directory when executing the command</description><displayName>Working Directory</displayName><dynamic>false</dynamic><name>Working Directory</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Argument Delimiter</key><value><defaultValue>;</defaultValue><description>Delimiter to use to separate arguments for a command [default: ;]. Must be a single character</description><displayName>Argument Delimiter</displayName><dynamic>false</dynamic><name>Argument Delimiter</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Output Destination Attribute</key><value><description>If set, the output of the stream command will be put into an attribute of the original FlowFile instead of a separate FlowFile. There will no longer be a relationship for 'output stream'. The value of this property will be the key for the output attribute.</description><displayName>Output Destination Attribute</displayName><dynamic>false</dynamic><name>Output Destination Attribute</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Max Attribute Length</key><value><defaultValue>256</defaultValue><description>If routing the output of the stream command to an attribute, the number of characters put to the attribute value will be at most this amount. This is important because attributes are held in memory and large attributes will quickly cause out of memory issues. If the output goes longer than this value, it will truncated to fit. Consider making this smaller if able.</description><displayName>Max Attribute Length</displayName><dynamic>false</dynamic><name>Max Attribute Length</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Command Arguments</key><value>${absolute.path};${filename};/import/TagListDataOnly.csv;/import/changed/;/import/final/;/import/intermediate/;</value></entry><entry><key>Command Path</key><value>/import/pythonscripts/ETL1.py</value></entry><entry><key>Ignore STDIN</key><value>false</value></entry><entry><key>Working Directory</key></entry><entry><key>Argument Delimiter</key><value>;</value></entry><entry><key>Output Destination Attribute</key><value>output stream</value></entry><entry><key>Max Attribute Length</key><value>256</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>ExecuteStreamCommand</name><relationships><autoTerminate>true</autoTerminate><description>FlowFiles that were successfully processed</description><name>original</name></relationships><state>STOPPED</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.ExecuteStreamCommand</type></processors><processors><id>9da7293b-c0c3-4db1-a55f-ca9efdde58ab</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><position><x>1317.5236438656214</x><y>819.8317389896154</y></position><config><bulletinLevel>DEBUG</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Input Directory</key><value><description>The input directory from which to pull files</description><displayName>Input Directory</displayName><dynamic>false</dynamic><name>Input Directory</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>File Filter</key><value><defaultValue>[^\.].*</defaultValue><description>Only files whose names match the given regular expression will be picked up</description><displayName>File Filter</displayName><dynamic>false</dynamic><name>File Filter</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Path Filter</key><value><description>When Recurse Subdirectories is true, then only subdirectories whose path matches the given regular expression will be scanned</description><displayName>Path Filter</displayName><dynamic>false</dynamic><name>Path Filter</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Batch Size</key><value><defaultValue>10</defaultValue><description>The maximum number of files to pull in each iteration</description><displayName>Batch Size</displayName><dynamic>false</dynamic><name>Batch Size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Keep Source File</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, the file is not deleted after it has been copied to the Content Repository; this causes the file to be picked up continually and is useful for testing purposes. If not keeping original NiFi will need write permissions on the directory it is pulling from otherwise it will ignore the file.</description><displayName>Keep Source File</displayName><dynamic>false</dynamic><name>Keep Source File</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Recurse Subdirectories</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>true</defaultValue><description>Indicates whether or not to pull files from subdirectories</description><displayName>Recurse Subdirectories</displayName><dynamic>false</dynamic><name>Recurse Subdirectories</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Polling Interval</key><value><defaultValue>0 sec</defaultValue><description>Indicates how long to wait before performing a directory listing</description><displayName>Polling Interval</displayName><dynamic>false</dynamic><name>Polling Interval</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Ignore Hidden Files</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>true</defaultValue><description>Indicates whether or not hidden files should be ignored</description><displayName>Ignore Hidden Files</displayName><dynamic>false</dynamic><name>Ignore Hidden Files</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Minimum File Age</key><value><defaultValue>0 sec</defaultValue><description>The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored</description><displayName>Minimum File Age</displayName><dynamic>false</dynamic><name>Minimum File Age</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Maximum File Age</key><value><description>The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored</description><displayName>Maximum File Age</displayName><dynamic>false</dynamic><name>Maximum File Age</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Minimum File Size</key><value><defaultValue>0 B</defaultValue><description>The minimum size that a file must be in order to be pulled</description><displayName>Minimum File Size</displayName><dynamic>false</dynamic><name>Minimum File Size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Maximum File Size</key><value><description>The maximum size that a file can be in order to be pulled</description><displayName>Maximum File Size</displayName><dynamic>false</dynamic><name>Maximum File Size</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Input Directory</key><value>/import/incoming/</value></entry><entry><key>File Filter</key><value>[^\.].*</value></entry><entry><key>Path Filter</key></entry><entry><key>Batch Size</key><value>10</value></entry><entry><key>Keep Source File</key><value>false</value></entry><entry><key>Recurse Subdirectories</key><value>true</value></entry><entry><key>Polling Interval</key><value>0 sec</value></entry><entry><key>Ignore Hidden Files</key><value>true</value></entry><entry><key>Minimum File Age</key><value>0 sec</value></entry><entry><key>Maximum File Age</key></entry><entry><key>Minimum File Size</key><value>0 B</value></entry><entry><key>Maximum File Size</key></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>GetFile_test2</name><relationships><autoTerminate>false</autoTerminate><description>All files are routed to success</description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>false</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.GetFile</type></processors><processors><id>c0115804-cd40-4174-8bbf-ece12e89965d</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><position><x>1324.7089596329763</x><y>1011.2782671487644</y></position><config><bulletinLevel>DEBUG</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Directory</key><value><description>The directory to which files should be written. You may use expression language such as /aa/bb/${path}</description><displayName>Directory</displayName><dynamic>false</dynamic><name>Directory</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Conflict Resolution Strategy</key><value><allowableValues><displayName>replace</displayName><value>replace</value></allowableValues><allowableValues><displayName>ignore</displayName><value>ignore</value></allowableValues><allowableValues><displayName>fail</displayName><value>fail</value></allowableValues><defaultValue>fail</defaultValue><description>Indicates what should happen when a file with the same name already exists in the output directory</description><displayName>Conflict Resolution Strategy</displayName><dynamic>false</dynamic><name>Conflict Resolution Strategy</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Create Missing Directories</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>true</defaultValue><description>If true, then missing destination directories will be created. If false, flowfiles are penalized and sent to failure.</description><displayName>Create Missing Directories</displayName><dynamic>false</dynamic><name>Create Missing Directories</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Maximum File Count</key><value><description>Specifies the maximum number of files that can exist in the output directory</description><displayName>Maximum File Count</displayName><dynamic>false</dynamic><name>Maximum File Count</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Last Modified Time</key><value><description>Sets the lastModifiedTime on the output file to the value of this attribute. Format must be yyyy-MM-dd'T'HH:mm:ssZ. You may also use expression language such as ${file.lastModifiedTime}.</description><displayName>Last Modified Time</displayName><dynamic>false</dynamic><name>Last Modified Time</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Permissions</key><value><description>Sets the permissions on the output file to the value of this attribute. Format must be either UNIX rwxrwxrwx with a - in place of denied permissions (e.g. rw-r--r--) or an octal number (e.g. 644). You may also use expression language such as ${file.permissions}.</description><displayName>Permissions</displayName><dynamic>false</dynamic><name>Permissions</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Owner</key><value><description>Sets the owner on the output file to the value of this attribute. You may also use expression language such as ${file.owner}.</description><displayName>Owner</displayName><dynamic>false</dynamic><name>Owner</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Group</key><value><description>Sets the group on the output file to the value of this attribute. You may also use expression language such as ${file.group}.</description><displayName>Group</displayName><dynamic>false</dynamic><name>Group</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Directory</key><value>/import/intermediate/</value></entry><entry><key>Conflict Resolution Strategy</key><value>replace</value></entry><entry><key>Create Missing Directories</key><value>true</value></entry><entry><key>Maximum File Count</key></entry><entry><key>Last Modified Time</key></entry><entry><key>Permissions</key></entry><entry><key>Owner</key></entry><entry><key>Group</key></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>PutFile</name><relationships><autoTerminate>false</autoTerminate><description>Files that could not be written to the output directory for some reason are transferred to this relationship</description><name>failure</name></relationships><relationships><autoTerminate>false</autoTerminate><description>Files that have been successfully written to the output directory are transferred to this relationship</description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>false</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.PutFile</type></processors><processors><id>47a10ff4-ea4c-47cd-a2dc-367f62f67705</id><parentGroupId>d6d2d32e-3e95-4465-b3d9-878a6b0a332c</parentGroupId><position><x>1310.1261354632097</x><y>1223.1939134218937</y></position><config><bulletinLevel>DEBUG</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Number of Relationships</key><value><defaultValue>1</defaultValue><description>Determines the number of Relationships to which the load should be distributed</description><displayName>Number of Relationships</displayName><dynamic>false</dynamic><name>Number of Relationships</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Distribution Strategy</key><value><allowableValues><displayName>round robin</displayName><value>round robin</value></allowableValues><allowableValues><displayName>next available</displayName><value>next available</value></allowableValues><allowableValues><displayName>load distribution service</displayName><value>load distribution service</value></allowableValues><defaultValue>round robin</defaultValue><description>Determines how the load will be distributed. If using Round Robin, will not distribute any FlowFiles unless all destinations can accept FlowFiles; when using Next Available, will distribute FlowFiles as long as at least 1 destination can accept FlowFiles.</description><displayName>Distribution Strategy</displayName><dynamic>false</dynamic><name>Distribution Strategy</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Number of Relationships</key><value>5</value></entry><entry><key>Distribution Strategy</key><value>round robin</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>DistributeLoad</name><relationships><autoTerminate>false</autoTerminate><description>Where to route flowfiles for this relationship index</description><name>1</name></relationships><relationships><autoTerminate>false</autoTerminate><description>Where to route flowfiles for this relationship index</description><name>2</name></relationships><relationships><autoTerminate>false</autoTerminate><description>Where to route flowfiles for this relationship index</description><name>3</name></relationships><relationships><autoTerminate>false</autoTerminate><description>Where to route flowfiles for this relationship index</description><name>4</name></relationships><relationships><autoTerminate>false</autoTerminate><description>Where to route flowfiles for this relationship index</description><name>5</name></relationships><state>STOPPED</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.DistributeLoad</type></processors></snippet><timestamp>04/25/2016 18:55:57 UTC</timestamp></template>
