Repository: nifi Updated Branches: refs/heads/master 9e7610ac7 -> 4112af013
NIFI-4130 Add lookup controller service in TransformXML to define XSLT from the UI addressed review comments Signed-off-by: Matthew Burgess <[email protected]> This closes #1953 Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/4112af01 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/4112af01 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/4112af01 Branch: refs/heads/master Commit: 4112af013d1b1f49e83f881d85ebe66e097840b5 Parents: 9e7610a Author: Pierre Villard <[email protected]> Authored: Mon Jun 26 17:48:06 2017 +0200 Committer: Matthew Burgess <[email protected]> Committed: Wed Nov 14 09:45:20 2018 -0500 ---------------------------------------------------------------------- .../nifi/processors/standard/TransformXml.java | 132 +++++++++++++++--- .../processors/standard/TestTransformXml.java | 133 ++++++++++++++++++- 2 files changed, 238 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/4112af01/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java index ff15428..1cc57fa 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TransformXml.java @@ -16,21 +16,24 @@ */ package org.apache.nifi.processors.standard; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + import javax.xml.XMLConstants; import javax.xml.transform.OutputKeys; import javax.xml.transform.Templates; @@ -39,6 +42,8 @@ import javax.xml.transform.TransformerConfigurationException; import javax.xml.transform.TransformerFactory; import javax.xml.transform.stream.StreamResult; import javax.xml.transform.stream.StreamSource; + +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.annotation.behavior.DynamicProperty; import org.apache.nifi.annotation.behavior.EventDriven; import org.apache.nifi.annotation.behavior.InputRequirement; @@ -49,6 +54,7 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.Tags; import org.apache.nifi.annotation.lifecycle.OnScheduled; import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; @@ -56,6 +62,9 @@ import org.apache.nifi.expression.AttributeExpression; import org.apache.nifi.expression.ExpressionLanguageScope; import org.apache.nifi.flowfile.FlowFile; import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.lookup.LookupFailureException; +import org.apache.nifi.lookup.LookupService; +import org.apache.nifi.lookup.StringLookupService; import org.apache.nifi.processor.AbstractProcessor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -67,6 +76,10 @@ import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.util.StopWatch; import org.apache.nifi.util.Tuple; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; + @EventDriven @SideEffectFree @SupportsBatching @@ -82,12 +95,33 @@ public class TransformXml extends AbstractProcessor { public static final PropertyDescriptor XSLT_FILE_NAME = new PropertyDescriptor.Builder() .name("XSLT file name") - .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content.") - .required(true) + .description("Provides the name (including full path) of the XSLT file to apply to the flowfile XML content." + + "One of the 'XSLT file name' and 'XSLT Lookup' properties must be defined.") + .required(false) .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) .addValidator(StandardValidators.FILE_EXISTS_VALIDATOR) .build(); + public static final PropertyDescriptor XSLT_CONTROLLER = new PropertyDescriptor.Builder() + .name("xslt-controller") + .displayName("XSLT Lookup") + .description("Controller lookup used to store XSLT definitions. One of the 'XSLT file name' and " + + "'XSLT Lookup' properties must be defined. WARNING: note that the lookup controller service " + + "should not be used to store large XSLT files.") + .required(false) + .identifiesControllerService(StringLookupService.class) + .build(); + + public static final PropertyDescriptor XSLT_CONTROLLER_KEY = new PropertyDescriptor.Builder() + .name("xslt-controller-key") + .displayName("XSLT Lookup key") + .description("Key used to retrieve the XSLT definition from the XSLT lookup controller. This property must be " + + "set when using the XSLT controller property.") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR) + .build(); + public static final PropertyDescriptor INDENT_OUTPUT = new PropertyDescriptor.Builder() .name("indent-output") .displayName("Indent") @@ -140,10 +174,14 @@ public class TransformXml extends AbstractProcessor { private Set<Relationship> relationships; private LoadingCache<String, Templates> cache; + private static AtomicReference<LookupService<String>> lookupService = new AtomicReference<LookupService<String>>(null); + @Override protected void init(final ProcessorInitializationContext context) { final List<PropertyDescriptor> properties = new ArrayList<>(); properties.add(XSLT_FILE_NAME); + properties.add(XSLT_CONTROLLER); + properties.add(XSLT_CONTROLLER_KEY); properties.add(INDENT_OUTPUT); properties.add(SECURE_PROCESSING); properties.add(CACHE_SIZE); @@ -167,6 +205,47 @@ public class TransformXml extends AbstractProcessor { } @Override + protected Collection<ValidationResult> customValidate(ValidationContext validationContext) { + final List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext)); + + PropertyValue filename = validationContext.getProperty(XSLT_FILE_NAME); + PropertyValue controller = validationContext.getProperty(XSLT_CONTROLLER); + PropertyValue key = validationContext.getProperty(XSLT_CONTROLLER_KEY); + + if((filename.isSet() && controller.isSet()) + || (!filename.isSet() && !controller.isSet())) { + results.add(new ValidationResult.Builder() + .valid(false) + .subject(this.getClass().getSimpleName()) + .explanation("Exactly one of the \"XSLT file name\" and \"XSLT controller\" properties must be defined.") + .build()); + } + + if(controller.isSet() && !key.isSet()) { + results.add(new ValidationResult.Builder() + .valid(false) + .subject(XSLT_CONTROLLER_KEY.getDisplayName()) + .explanation("If using \"XSLT controller\", the XSLT controller key property must be defined.") + .build()); + } + + if(controller.isSet()) { + final LookupService<String> lookupService = validationContext.getProperty(XSLT_CONTROLLER).asControllerService(StringLookupService.class); + final Set<String> requiredKeys = lookupService.getRequiredKeys(); + if (requiredKeys == null || requiredKeys.size() != 1) { + results.add(new ValidationResult.Builder() + .valid(false) + .subject(XSLT_CONTROLLER.getDisplayName()) + .explanation("This processor requires a key-value lookup service supporting exactly one required key, was: " + + (requiredKeys == null ? "null" : String.valueOf(requiredKeys.size()))) + .build()); + } + } + + return results; + } + + @Override protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { return new PropertyDescriptor.Builder() .name(propertyDescriptorName) @@ -177,9 +256,10 @@ public class TransformXml extends AbstractProcessor { .build(); } - private Templates newTemplates(ProcessContext context, String path) throws TransformerConfigurationException { + private Templates newTemplates(final ProcessContext context, final String path) throws TransformerConfigurationException, LookupFailureException { final Boolean secureProcessing = context.getProperty(SECURE_PROCESSING).asBoolean(); TransformerFactory factory = TransformerFactory.newInstance(); + final boolean isFilename = context.getProperty(XSLT_FILE_NAME).isSet(); if (secureProcessing) { factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); @@ -188,7 +268,17 @@ public class TransformXml extends AbstractProcessor { factory.setFeature("http://saxon.sf.net/feature/parserFeature?uri=http://xml.org/sax/features/external-general-entities";, false); } - return factory.newTemplates(new StreamSource(path)); + if(isFilename) { + return factory.newTemplates(new StreamSource(path)); + } else { + final String coordinateKey = lookupService.get().getRequiredKeys().iterator().next(); + final Optional<String> attributeValue = lookupService.get().lookup(Collections.singletonMap(coordinateKey, path)); + if (attributeValue.isPresent() && StringUtils.isNotBlank(attributeValue.get())) { + return factory.newTemplates(new StreamSource(new ByteArrayInputStream(attributeValue.get().getBytes(StandardCharsets.UTF_8)))); + } else { + throw new TransformerConfigurationException("No XSLT definition is associated to " + path + " in the lookup controller service."); + } + } } @OnScheduled @@ -198,20 +288,21 @@ public class TransformXml extends AbstractProcessor { final Long cacheTTL = context.getProperty(CACHE_TTL_AFTER_LAST_ACCESS).asTimePeriod(TimeUnit.SECONDS); if (cacheSize > 0) { - CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize); + CacheBuilder<Object, Object> cacheBuilder = CacheBuilder.newBuilder().maximumSize(cacheSize); if (cacheTTL > 0) { cacheBuilder = cacheBuilder.expireAfterAccess(cacheTTL, TimeUnit.SECONDS); } cache = cacheBuilder.build( - new CacheLoader<String, Templates>() { - public Templates load(String path) throws TransformerConfigurationException { - return newTemplates(context, path); - } - }); + new CacheLoader<String, Templates>() { + @Override + public Templates load(String path) throws TransformerConfigurationException, LookupFailureException { + return newTemplates(context, path); + } + }); } else { cache = null; - logger.warn("Stylesheet cache disabled because cache size is set to 0"); + logger.info("Stylesheet cache disabled because cache size is set to 0"); } } @@ -224,10 +315,11 @@ public class TransformXml extends AbstractProcessor { final ComponentLog logger = getLogger(); final StopWatch stopWatch = new StopWatch(true); - final String xsltFileName = context.getProperty(XSLT_FILE_NAME) - .evaluateAttributeExpressions(original) - .getValue(); + final String path = context.getProperty(XSLT_FILE_NAME).isSet() + ? context.getProperty(XSLT_FILE_NAME).evaluateAttributeExpressions(original).getValue() + : context.getProperty(XSLT_CONTROLLER_KEY).evaluateAttributeExpressions(original).getValue(); final Boolean indentOutput = context.getProperty(INDENT_OUTPUT).asBoolean(); + lookupService.set(context.getProperty(XSLT_CONTROLLER).asControllerService(LookupService.class)); try { FlowFile transformed = session.write(original, new StreamCallback() { @@ -236,9 +328,9 @@ public class TransformXml extends AbstractProcessor { try (final InputStream in = new BufferedInputStream(rawIn)) { final Templates templates; if (cache != null) { - templates = cache.get(xsltFileName); + templates = cache.get(path); } else { - templates = newTemplates(context, xsltFileName); + templates = newTemplates(context, path); } final Transformer transformer = templates.newTransformer(); @@ -303,4 +395,4 @@ public class TransformXml extends AbstractProcessor { } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/4112af01/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java index 5ced266..3d5f1e0 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTransformXml.java @@ -16,23 +16,21 @@ */ package org.apache.nifi.processors.standard; -import java.io.IOException; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import org.apache.nifi.lookup.SimpleKeyValueLookupService; +import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.util.MockFlowFile; import org.apache.nifi.util.TestRunner; import org.apache.nifi.util.TestRunners; - import org.junit.Test; public class TestTransformXml { @@ -56,8 +54,6 @@ public class TestTransformXml { runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE); final MockFlowFile original = runner.getFlowFilesForRelationship(TransformXml.REL_FAILURE).get(0); - final String originalContent = new String(original.toByteArray(), StandardCharsets.UTF_8); - original.assertContentEquals("not xml"); } @@ -107,7 +103,6 @@ public class TestTransformXml { runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); - final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1); final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/tokens.xml"))); transformed.assertContentEquals(expectedContent); @@ -148,4 +143,128 @@ public class TestTransformXml { transformed.assertContentEquals(expectedContent); } + @Test + public void testTransformBothControllerFileNotValid() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + runner.setProperty(TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/math.xsl"); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "key1", "value1"); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + + runner.assertNotValid(); + } + + @Test + public void testTransformNoneControllerFileNotValid() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + runner.setProperty(TransformXml.CACHE_SIZE, "0"); + runner.assertNotValid(); + } + + @Test + public void testTransformControllerNoKey() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "key1", "value1"); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + + runner.assertNotValid(); + } + + @Test + public void testTransformWithController() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "math", "<xsl:stylesheet version=\"2.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\"; xmlns:xs=\"http://www.w3.org/2001/XMLSchema\";>" + + "<xsl:param name=\"header\" /><xsl:template match=\"doc\">" + + "<HTML><H1><xsl:value-of select=\"$header\"/></H1><HR/>" + + "<P>Should say \"1\": <xsl:value-of select=\"5 mod 2\"/></P>" + + "<P>Should say \"1\": <xsl:value-of select=\"n1 mod n2\"/></P>" + + "<P>Should say \"-1\": <xsl:value-of select=\"div mod mod\"/></P>" + + "<P><xsl:value-of select=\"div or ((mod)) | or\"/></P>" + + "</HTML></xsl:template></xsl:stylesheet>"); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}"); + runner.setProperty("header", "Test for mod"); + + final Map<String, String> attributes = new HashMap<>(); + attributes.put("xslt", "math"); + runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); + final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim(); + + transformed.assertContentEquals(expectedContent); + } + + @Test + public void testTransformWithXsltNotFoundInController() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}"); + runner.setProperty("header", "Test for mod"); + + final Map<String, String> attributes = new HashMap<>(); + attributes.put("xslt", "math"); + runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE); + } + + @Test + public void testTransformWithControllerNoCache() throws IOException, InitializationException { + final TestRunner runner = TestRunners.newTestRunner(new TransformXml()); + + final SimpleKeyValueLookupService service = new SimpleKeyValueLookupService(); + runner.addControllerService("simple-key-value-lookup-service", service); + runner.setProperty(service, "math", "<xsl:stylesheet version=\"2.0\" xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\"; xmlns:xs=\"http://www.w3.org/2001/XMLSchema\";>" + + "<xsl:param name=\"header\" /><xsl:template match=\"doc\">" + + "<HTML><H1><xsl:value-of select=\"$header\"/></H1><HR/>" + + "<P>Should say \"1\": <xsl:value-of select=\"5 mod 2\"/></P>" + + "<P>Should say \"1\": <xsl:value-of select=\"n1 mod n2\"/></P>" + + "<P>Should say \"-1\": <xsl:value-of select=\"div mod mod\"/></P>" + + "<P><xsl:value-of select=\"div or ((mod)) | or\"/></P>" + + "</HTML></xsl:template></xsl:stylesheet>"); + runner.enableControllerService(service); + runner.assertValid(service); + runner.setProperty(TransformXml.XSLT_CONTROLLER, "simple-key-value-lookup-service"); + runner.setProperty(TransformXml.XSLT_CONTROLLER_KEY, "${xslt}"); + runner.setProperty(TransformXml.CACHE_SIZE, "0"); + runner.setProperty("header", "Test for mod"); + + final Map<String, String> attributes = new HashMap<>(); + attributes.put("xslt", "math"); + runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes); + + runner.run(); + + runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS); + final MockFlowFile transformed = runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0); + final String expectedContent = new String(Files.readAllBytes(Paths.get("src/test/resources/TestTransformXml/math.html"))).trim(); + + transformed.assertContentEquals(expectedContent); + } + }
