This is an automated email from the ASF dual-hosted git repository.
mattyb149 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/master by this push:
new c93ee5a NIFI-6186 Resolve handling of module paths in
JythonScriptEngineConfigurator
c93ee5a is described below
commit c93ee5ad5949858bd3f4eec2067ae30016ea76ac
Author: ambah <[email protected]>
AuthorDate: Wed Apr 3 18:15:58 2019 -0400
NIFI-6186 Resolve handling of module paths in JythonScriptEngineConfigurator
Signed-off-by: Matthew Burgess <[email protected]>
This closes #3406
---
.../impl/JythonScriptEngineConfigurator.java | 3 +-
.../nifi/processors/script/TestInvokeJython.java | 55 ++++++++++++++++
.../src/test/resources/jython/test_modules_path.py | 74 ++++++++++++++++++++++
3 files changed, 131 insertions(+), 1 deletion(-)
diff --git
a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptEngineConfigurator.java
b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptEngineConfigurator.java
index 458658b..3b8271b 100644
---
a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptEngineConfigurator.java
+++
b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptEngineConfigurator.java
@@ -18,6 +18,7 @@ package org.apache.nifi.script.impl;
import org.apache.nifi.logging.ComponentLog;
import org.apache.nifi.processors.script.ScriptEngineConfigurator;
+import org.python.core.PyString;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
@@ -47,7 +48,7 @@ public class JythonScriptEngineConfigurator implements
ScriptEngineConfigurator
engine.eval("import sys");
if (modulePaths != null) {
for (String modulePath : modulePaths) {
- engine.eval("sys.path.append('" + modulePath + "')");
+ engine.eval("sys.path.append(" +
PyString.encode_UnicodeEscape(modulePath, true) + ")");
}
}
}
diff --git
a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeJython.java
b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeJython.java
index 5b17e04..345f4af 100755
---
a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeJython.java
+++
b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeJython.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.processors.script;
+import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.script.ScriptingComponentUtils;
import org.apache.nifi.util.MockFlowFile;
@@ -31,6 +32,9 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
public class TestInvokeJython extends BaseScriptTest {
@@ -95,6 +99,57 @@ public class TestInvokeJython extends BaseScriptTest {
}
/**
+ * Test a script that has a Jython processor that reads the system path as
controlled by the Module Directory property then stores it in the attributes of
the flowfile being routed.
+ * <p>
+ * This tests whether the JythonScriptEngineConfigurator successfully
translates the "Module Directory" property into Python system paths, even with
strings that contain Python escape sequences
+ *
+ * @throws Exception Any error encountered while testing
+ */
+ @Test
+ public void testUpdateAttributeFromProcessorModulePaths() throws Exception
{
+ // Prepare a set of easily identified paths for the Module Directory
property
+ final String moduleDirectoryTestPrefix = "test";
+ final String[] testModuleDirectoryValues = {
"abc","\\a\\b\\c","\\123","\\d\"e" };
+ final int numTestValues = testModuleDirectoryValues.length;
+ // Prepend each module directory value with a simple prefix and an
identifying number so we can identify it later.
+ final List<String> testModuleDirectoryFullValues =
IntStream.range(0,numTestValues)
+ .boxed()
+ .map(i ->
String.format("%s#%s#%s",moduleDirectoryTestPrefix,i,testModuleDirectoryValues[i]))
+ .collect(Collectors.toList());
+ final String testModuleDirectoryCombined =
String.join(",",testModuleDirectoryFullValues);
+
+ // Run the script that captures the system path resulting from the
Module Directory property
+ final TestRunner runner = TestRunners.newTestRunner(new
InvokeScriptedProcessor());
+
+ runner.setValidateExpressionUsage(false);
+
runner.setProperty(scriptingComponent.getScriptingComponentHelper().SCRIPT_ENGINE,
"python");
+ runner.setProperty(ScriptingComponentUtils.SCRIPT_FILE,
"target/test/resources/jython/test_modules_path.py");
+ runner.setProperty(ScriptingComponentUtils.MODULES,
testModuleDirectoryCombined);
+
+ final Map<String, String> attributes = new HashMap<>();
+
+ runner.assertValid();
+ runner.enqueue(new byte[0], attributes);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred("success", 1);
+ final List<MockFlowFile> result =
runner.getFlowFilesForRelationship("success");
+
+ // verify successful processing of the module paths
+ result.get(0).assertAttributeExists("from-path");
+ final String[] effectivePaths =
result.get(0).getAttribute("from-path").split(","); // Extract the
comma-delimited paths from the script-produced attribute
+ Assert.assertTrue(effectivePaths.length >= numTestValues); // we
should have our test values, plus defaults
+ // Isolate only the paths with our identified prefix
+ final List<String> relevantPaths =
Arrays.stream(effectivePaths).filter(path ->
path.startsWith(moduleDirectoryTestPrefix)).collect(Collectors.toList());
+ Assert.assertEquals(testModuleDirectoryFullValues.size(),
relevantPaths.size());
+ relevantPaths.forEach(path -> {
+ final int resultIx =
Integer.valueOf(StringUtils.substringBetween(path,"#")); // extract the index
so we can relate it to the sources, despite potential mangling
+ final String expectedValue =
testModuleDirectoryFullValues.get(resultIx);
+ Assert.assertEquals(expectedValue, path); // Ensure our path was
passed through without mangling
+ });
+ }
+
+ /**
* Tests a script that has a Jython Processor that that reads the first
line of text from the flowfiles content and stores the value in an attribute of
the outgoing flowfile.
*
* @throws Exception Any error encountered while testing
diff --git
a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/jython/test_modules_path.py
b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/jython/test_modules_path.py
new file mode 100644
index 0000000..bee52e6
--- /dev/null
+++
b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/resources/jython/test_modules_path.py
@@ -0,0 +1,74 @@
+#! /usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import sys
+import traceback
+from org.apache.nifi.processor import Processor
+from org.apache.nifi.processor import Relationship
+
+class ReadModulesPathAndStoreAsAttribute(Processor) :
+ __rel_success =
Relationship.Builder().description("Success").name("success").build()
+
+ def __init__(self) :
+ pass
+
+ def initialize(self, context) :
+ pass
+
+ def getRelationships(self) :
+ return set([self.__rel_success])
+
+ def validate(self, context) :
+ pass
+
+ def getPropertyDescriptors(self) :
+ pass
+
+ def onPropertyModified(self, descriptor, newValue, oldValue) :
+ pass
+
+ def onTrigger(self, context, sessionFactory) :
+ session = sessionFactory.createSession()
+ try :
+ # ensure there is work to do
+ flowfile = session.get()
+ if flowfile is None :
+ return
+
+ # Extract sys.path and encode as a comma-separated string
+ sysPathString = ','.join(sys.path)
+
+ # set an attribute with the captured contents of sys.path
+ flowfile = session.putAttribute(flowfile, "from-path",
sysPathString)
+
+ # transfer
+ session.transfer(flowfile, self.__rel_success)
+ session.commit()
+ except :
+ print sys.exc_info()[0]
+ print "Exception in TestReader:"
+ print '-' * 60
+ traceback.print_exc(file=sys.stdout)
+ print '-' * 60
+
+ session.rollback(True)
+ raise
+
+processor = ReadModulesPathAndStoreAsAttribute()
\ No newline at end of file
