alopresto commented on issue #4147: NIFI-7260: Fix error handling and re-evaluate Module Directory property on changed for scripted controller services URL: https://github.com/apache/nifi/pull/4147#issuecomment-599829133 Hi Matt, the code all seems to make sense but when I tested it out with a simple `GFF` -> `QueryRecord` -> `LogAttribute` flow, I get an error enabling the `ScriptedReader` (using the same script you provide for testing) which doesn't block enabling the `QR` processor, which I thought was the whole point of the PR? Hopefully I'm just doing something wrong? I've attached the flow definition as it exists. ``` <?xml version="1.0" encoding="UTF-8" standalone="no"?> <flowController encoding-version="1.4"> <maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount> <maxEventDrivenThreadCount>1</maxEventDrivenThreadCount> <registries/> <parameterContexts/> <rootGroup> <id>e60749ef-0170-1000-f44e-ff67968bebad</id> <name>NiFi Flow</name> <position x="0.0" y="0.0"/> <comment/> <processor> <id>e60aa427-0170-1000-a067-6c850d09968d</id> <name>GenerateFlowFile</name> <position x="498.0" y="179.0"/> <styles/> <comment/> <class>org.apache.nifi.processors.standard.GenerateFlowFile</class> <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> <version>1.12.0-SNAPSHOT</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>1 sec</schedulingPeriod> <penalizationPeriod>30 sec</penalizationPeriod> <yieldPeriod>1 sec</yieldPeriod> <bulletinLevel>WARN</bulletinLevel> <lossTolerant>false</lossTolerant> <scheduledState>STOPPED</scheduledState> <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> <executionNode>ALL</executionNode> <runDurationNanos>0</runDurationNanos> <property> <name>File Size</name> <value>0B</value> </property> <property> <name>Batch Size</name> <value>1</value> </property> <property> <name>Data Format</name> <value>Text</value> </property> <property> <name>Unique FlowFiles</name> <value>false</value> </property> <property> <name>generate-ff-custom-text</name> </property> <property> <name>character-set</name> <value>UTF-8</value> </property> <property> <name>mime-type</name> </property> </processor> <processor> <id>e60cb66f-0170-1000-d8c3-7b9cfcac060e</id> <name>LogAttribute</name> <position x="496.0" y="704.0"/> <styles/> <comment/> <class>org.apache.nifi.processors.standard.LogAttribute</class> <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> <version>1.12.0-SNAPSHOT</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> <penalizationPeriod>30 sec</penalizationPeriod> <yieldPeriod>1 sec</yieldPeriod> <bulletinLevel>WARN</bulletinLevel> <lossTolerant>false</lossTolerant> <scheduledState>DISABLED</scheduledState> <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> <executionNode>ALL</executionNode> <runDurationNanos>0</runDurationNanos> <property> <name>Log Level</name> <value>info</value> </property> <property> <name>Log Payload</name> <value>false</value> </property> <property> <name>Attributes to Log</name> </property> <property> <name>attributes-to-log-regex</name> <value>.*</value> </property> <property> <name>Attributes to Ignore</name> </property> <property> <name>attributes-to-ignore-regex</name> </property> <property> <name>Log prefix</name> </property> <property> <name>character-set</name> <value>UTF-8</value> </property> </processor> <processor> <id>e60b1443-0170-1000-77f9-682b26bcba4c</id> <name>QueryRecord</name> <position x="496.0" y="432.0"/> <styles/> <comment/> <class>org.apache.nifi.processors.standard.QueryRecord</class> <bundle> <group>org.apache.nifi</group> <artifact>nifi-standard-nar</artifact> <version>1.12.0-SNAPSHOT</version> </bundle> <maxConcurrentTasks>1</maxConcurrentTasks> <schedulingPeriod>0 sec</schedulingPeriod> <penalizationPeriod>30 sec</penalizationPeriod> <yieldPeriod>1 sec</yieldPeriod> <bulletinLevel>WARN</bulletinLevel> <lossTolerant>false</lossTolerant> <scheduledState>STOPPED</scheduledState> <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> <executionNode>ALL</executionNode> <runDurationNanos>0</runDurationNanos> <property> <name>record-reader</name> <value>e60b4c69-0170-1000-b09b-0ac9ef941c55</value> </property> <property> <name>record-writer</name> <value>e60b77f3-0170-1000-18d5-06515ce81d48</value> </property> <property> <name>include-zero-record-flowfiles</name> <value>true</value> </property> <property> <name>cache-schema</name> <value>true</value> </property> <property> <name>Name</name> <value>SELECT name FROM FLOWFILE</value> </property> </processor> <connection> <id>e60dcafa-0170-1000-a59f-a40c85941004</id> <name/> <bendPoints> <bendPoint x="472.0" y="632.0"/> </bendPoints> <labelIndex>1</labelIndex> <zIndex>0</zIndex> <sourceId>e60b1443-0170-1000-77f9-682b26bcba4c</sourceId> <sourceGroupId>e60749ef-0170-1000-f44e-ff67968bebad</sourceGroupId> <sourceType>PROCESSOR</sourceType> <destinationId>e60cb66f-0170-1000-d8c3-7b9cfcac060e</destinationId> <destinationGroupId>e60749ef-0170-1000-f44e-ff67968bebad</destinationGroupId> <destinationType>PROCESSOR</destinationType> <relationship>Name</relationship> <maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> <flowFileExpiration>0 sec</flowFileExpiration> <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> <partitioningAttribute/> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> <connection> <id>e60cd44b-0170-1000-9753-597cb32bc4f5</id> <name/> <bendPoints> <bendPoint x="848.0" y="632.0"/> </bendPoints> <labelIndex>0</labelIndex> <zIndex>0</zIndex> <sourceId>e60b1443-0170-1000-77f9-682b26bcba4c</sourceId> <sourceGroupId>e60749ef-0170-1000-f44e-ff67968bebad</sourceGroupId> <sourceType>PROCESSOR</sourceType> <destinationId>e60cb66f-0170-1000-d8c3-7b9cfcac060e</destinationId> <destinationGroupId>e60749ef-0170-1000-f44e-ff67968bebad</destinationGroupId> <destinationType>PROCESSOR</destinationType> <relationship>original</relationship> <relationship>failure</relationship> <maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> <flowFileExpiration>0 sec</flowFileExpiration> <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> <partitioningAttribute/> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> <connection> <id>e60b8cee-0170-1000-8dbe-05792b5f420e</id> <name/> <bendPoints/> <labelIndex>1</labelIndex> <zIndex>0</zIndex> <sourceId>e60aa427-0170-1000-a067-6c850d09968d</sourceId> <sourceGroupId>e60749ef-0170-1000-f44e-ff67968bebad</sourceGroupId> <sourceType>PROCESSOR</sourceType> <destinationId>e60b1443-0170-1000-77f9-682b26bcba4c</destinationId> <destinationGroupId>e60749ef-0170-1000-f44e-ff67968bebad</destinationGroupId> <destinationType>PROCESSOR</destinationType> <relationship>success</relationship> <maxWorkQueueSize>10000</maxWorkQueueSize> <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> <flowFileExpiration>0 sec</flowFileExpiration> <loadBalanceStrategy>DO_NOT_LOAD_BALANCE</loadBalanceStrategy> <partitioningAttribute/> <loadBalanceCompression>DO_NOT_COMPRESS</loadBalanceCompression> </connection> <controllerService> <id>e60b4c69-0170-1000-b09b-0ac9ef941c55</id> <name>ScriptedReader</name> <comment/> <class>org.apache.nifi.record.script.ScriptedReader</class> <bundle> <group>org.apache.nifi</group> <artifact>nifi-scripting-nar</artifact> <version>1.12.0-SNAPSHOT</version> </bundle> <enabled>true</enabled> <property> <name>Script Engine</name> <value>Groovy</value> </property> <property> <name>Script File</name> </property> <property> <name>Script Body</name> <value>import org.apache.nifi.controller.AbstractControllerService import org.apache.nifi.logging.ComponentLog import org.apache.nifi.schema.access.SchemaNotFoundException import org.apache.nifi.serialization.MalformedRecordException import org.apache.nifi.serialization.RecordReader import org.apache.nifi.serialization.RecordReaderFactory import org.apache.nifi.serialization.SimpleRecordSchema import org.apache.nifi.serialization.record.MapRecord import org.apache.nifi.serialization.record.Record import org.apache.nifi.serialization.record.RecordField import org.apache.nifi.serialization.record.RecordFieldType import org.apache.nifi.serialization.record.RecordSchema // import a test class to ensure Module Directory property is working correctly import org.apache.nifi.script.ModulePropertyExample class GroovyModuleRecordReader implements RecordReader { def recordSchema = new SimpleRecordSchema( [new RecordField('id', RecordFieldType.INT.dataType), new RecordField('name', RecordFieldType.STRING.dataType), new RecordField('code', RecordFieldType.INT.dataType)] ) def recordIterator = [ new MapRecord(recordSchema, ['id': 1, 'name': 'John', 'code': 100]), new MapRecord(recordSchema, ['id': 2, 'name': 'Mary', 'code': 200]), new MapRecord(recordSchema, ['id': 3, 'name': 'Ramon', 'code': 300]) ].iterator() Record nextRecord(boolean coerceTypes, boolean dropUnknown) throws IOException, MalformedRecordException { return recordIterator.hasNext() ? recordIterator.next() : null } RecordSchema getSchema() throws MalformedRecordException { return recordSchema } void close() throws IOException { } } class GroovyModuleRecordReaderFactory extends AbstractControllerService implements RecordReaderFactory { RecordReader createRecordReader(Map<String, String> variables, InputStream inputStream, long inputLength, ComponentLog logger) throws MalformedRecordException, IOException, SchemaNotFoundException { return new GroovyModuleRecordReader() } } reader = new GroovyModuleRecordReaderFactory()</value> </property> <property> <name>Module Directory</name> </property> </controllerService> <controllerService> <id>e60b77f3-0170-1000-18d5-06515ce81d48</id> <name>JsonRecordSetWriter</name> <comment/> <class>org.apache.nifi.json.JsonRecordSetWriter</class> <bundle> <group>org.apache.nifi</group> <artifact>nifi-record-serialization-services-nar</artifact> <version>1.12.0-SNAPSHOT</version> </bundle> <enabled>true</enabled> <property> <name>Schema Write Strategy</name> <value>no-schema</value> </property> <property> <name>schema-cache</name> </property> <property> <name>schema-access-strategy</name> <value>inherit-record-schema</value> </property> <property> <name>schema-registry</name> </property> <property> <name>schema-name</name> <value>${schema.name}</value> </property> <property> <name>schema-version</name> </property> <property> <name>schema-branch</name> </property> <property> <name>schema-text</name> <value>${avro.schema}</value> </property> <property> <name>Date Format</name> </property> <property> <name>Time Format</name> </property> <property> <name>Timestamp Format</name> </property> <property> <name>Pretty Print JSON</name> <value>false</value> </property> <property> <name>suppress-nulls</name> <value>never-suppress</value> </property> <property> <name>output-grouping</name> <value>output-array</value> </property> <property> <name>compression-format</name> <value>none</value> </property> <property> <name>compression-level</name> <value>1</value> </property> </controllerService> </rootGroup> <controllerServices/> <reportingTasks/> </flowController> ``` And the log output: ``` 2020-03-16 18:16:38,401 ERROR [Timer-Driven Process Thread-10] o.a.n.c.s.StandardControllerServiceNode StandardControllerServiceNode[service=ScriptedReader[id=e60b4c69-0170-1000-b09b-0ac9ef941c55], versionedComponentId=null, processGroup=StandardProcessGroup[identifier=e60749ef-0170-1000-f44e-ff67968bebad], active=true] Failed to invoke @OnEnabled method due to org.apache.nifi.processor.exception.ProcessException: Error during loading of script: {} org.apache.nifi.processor.exception.ProcessException: Error during loading of script at org.apache.nifi.record.script.AbstractScriptedRecordFactory.setup(AbstractScriptedRecordFactory.java:50) at org.apache.nifi.script.AbstractScriptedControllerService.onEnabled(AbstractScriptedControllerService.java:143) at org.apache.nifi.record.script.ScriptedReader.onEnabled(ScriptedReader.java:59) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotations(ReflectionUtils.java:142) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotations(ReflectionUtils.java:130) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotations(ReflectionUtils.java:75) at org.apache.nifi.util.ReflectionUtils.invokeMethodsWithAnnotation(ReflectionUtils.java:52) at org.apache.nifi.controller.service.StandardControllerServiceNode$2.run(StandardControllerServiceNode.java:432) at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2020-03-16 18:16:38,401 ERROR [Timer-Driven Process Thread-10] o.a.n.c.s.StandardControllerServiceNode Failed to invoke @OnEnabled method of ScriptedReader[id=e60b4c69-0170-1000-b09b-0ac9ef941c55] due to org.apache.nifi.processor.exception.ProcessException: Error during loading of script ``` <img width="2032" alt="Screen Shot 2020-03-16 at 6 14 37 PM" src="https://user-images.githubusercontent.com/798465/76812470-32107380-67b2-11ea-8365-e6414e51c912.png";> <img width="2032" alt="Screen Shot 2020-03-16 at 6 14 40 PM" src="https://user-images.githubusercontent.com/798465/76812472-3341a080-67b2-11ea-8d5a-aa4a7e338b8c.png";>
---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
![https://user-images.githubusercontent.com/798465/76812470-32107380-67b2-11ea-8365-e6414e51c912.png"](https://user-images.githubusercontent.com/798465/76812470-32107380-67b2-11ea-8365-e6414e51c912.png"){kind=link}
![https://user-images.githubusercontent.com/798465/76812472-3341a080-67b2-11ea-8d5a-aa4a7e338b8c.png"](https://user-images.githubusercontent.com/798465/76812472-3341a080-67b2-11ea-8d5a-aa4a7e338b8c.png"){kind=link}