Repository: nifi Updated Branches: refs/heads/master c3754c392 -> 0536c3edf
NIFI-4353 Implemented new JAXB logic. Added unit test and template test resource. Added RAT exclusion. This closes #2128 Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/0536c3ed Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/0536c3ed Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/0536c3ed Branch: refs/heads/master Commit: 0536c3edf146b27c50e26c026e4cce2c1624acbd Parents: c3754c3 Author: Andy LoPresto <[email protected]> Authored: Tue Sep 5 18:50:12 2017 -0700 Committer: Matt Gilman <[email protected]> Committed: Wed Sep 6 13:13:24 2017 -0400 ---------------------------------------------------------------------- .../nifi-framework/nifi-framework-core/pom.xml | 1 + .../nifi/persistence/TemplateDeserializer.java | 16 +- .../persistence/TemplateDeserializerTest.groovy | 66 ++++++ .../src/test/resources/xxe_template.xml | 230 +++++++++++++++++++ 4 files changed, 309 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/0536c3ed/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml index 9d00f49..4989be6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml @@ -205,6 +205,7 @@ <exclude>src/test/resources/hello.txt</exclude> <exclude>src/test/resources/bye.txt</exclude> <exclude>src/test/resources/old-swap-file.swap</exclude> + <exclude>src/test/resources/xxe_template.xml</exclude> </excludes> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/nifi/blob/0536c3ed/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/TemplateDeserializer.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/TemplateDeserializer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/TemplateDeserializer.java index fef0709..27e9093 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/TemplateDeserializer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/persistence/TemplateDeserializer.java @@ -17,13 +17,14 @@ package org.apache.nifi.persistence; import java.io.InputStream; - import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBElement; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; import javax.xml.transform.stream.StreamSource; - import org.apache.nifi.controller.serialization.FlowSerializationException; import org.apache.nifi.web.api.dto.TemplateDTO; @@ -32,10 +33,17 @@ public class TemplateDeserializer { public static TemplateDTO deserialize(final InputStream inStream) { try { JAXBContext context = JAXBContext.newInstance(TemplateDTO.class); + + // Manually constructing the XIF is necessary to prevent XXE attacks + XMLInputFactory xif = XMLInputFactory.newFactory(); + xif.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + xif.setProperty(XMLInputFactory.SUPPORT_DTD, false); + XMLStreamReader xsr = xif.createXMLStreamReader(new StreamSource(inStream)); + Unmarshaller unmarshaller = context.createUnmarshaller(); - JAXBElement<TemplateDTO> templateElement = unmarshaller.unmarshal(new StreamSource(inStream), TemplateDTO.class); + JAXBElement<TemplateDTO> templateElement = unmarshaller.unmarshal(xsr, TemplateDTO.class); return templateElement.getValue(); - } catch (final JAXBException e) { + } catch (final JAXBException | XMLStreamException e) { throw new FlowSerializationException(e); } } http://git-wip-us.apache.org/repos/asf/nifi/blob/0536c3ed/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/persistence/TemplateDeserializerTest.groovy ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/persistence/TemplateDeserializerTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/persistence/TemplateDeserializerTest.groovy new file mode 100644 index 0000000..c90868b --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/groovy/org/apache/nifi/persistence/TemplateDeserializerTest.groovy @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.persistence + +import org.apache.nifi.web.api.dto.TemplateDTO +import org.junit.After +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +@RunWith(JUnit4.class) +class TemplateDeserializerTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(TemplateDeserializerTest.class) + + @BeforeClass + static void setUpOnce() throws Exception { + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + } + + @Before + void setUp() throws Exception { + + } + + @After + void tearDown() throws Exception { + + } + + @Test + void testShouldHandleXXEInTemplateLoad() { + // Arrange + final String XXE_TEMPLATE_FILEPATH = "src/test/resources/xxe_template.xml" + InputStream templateStream = new File(XXE_TEMPLATE_FILEPATH).newInputStream() + + // Act + def msg = shouldFail() { + TemplateDTO template = TemplateDeserializer.deserialize(templateStream) + logger.info("Deserialized template \"${template.name}\" -- ${template.description}") + } + + // Assert + logger.expected(msg) + assert msg =~ "XMLStreamException: ParseError " + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/0536c3ed/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/xxe_template.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/xxe_template.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/xxe_template.xml new file mode 100644 index 0000000..82674e0 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/xxe_template.xml @@ -0,0 +1,230 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?><!DOCTYPE netspi [<!ENTITY xxe SYSTEM "file:///etc/passwd" >]> +<template> + <name>&xxe;</name> + <description>A simple template which generates flowfiles and logs them. </description> + <groupId>3a204982-015e-1000-eaa2-19d352ec8394</groupId> + <snippet> + <connections> + <id>0fbe8be5-306c-3b6c-0000-000000000000</id> + <parentGroupId>21ae0bd6-5db6-3a47-0000-000000000000</parentGroupId> + <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold> + <backPressureObjectThreshold>10000</backPressureObjectThreshold> + <destination> + <groupId>21ae0bd6-5db6-3a47-0000-000000000000</groupId> + <id>fd90023d-a235-30f6-0000-000000000000</id> + <type>PROCESSOR</type> + </destination> + <flowFileExpiration>0 sec</flowFileExpiration> + <labelIndex>1</labelIndex> + <name></name> + <selectedRelationships>success</selectedRelationships> + <source> + <groupId>21ae0bd6-5db6-3a47-0000-000000000000</groupId> + <id>ff49910d-06bb-37ee-0000-000000000000</id> + <type>PROCESSOR</type> + </source> + <zIndex>0</zIndex> + </connections> + <processors> + <id>fd90023d-a235-30f6-0000-000000000000</id> + <parentGroupId>21ae0bd6-5db6-3a47-0000-000000000000</parentGroupId> + <position> + <x>0.0</x> + <y>318.3128613789876</y> + </position> + <bundle> + <artifact>nifi-standard-nar</artifact> + <group>org.apache.nifi</group> + <version>1.4.0-SNAPSHOT</version> + </bundle> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>Log Level</key> + <value> + <name>Log Level</name> + </value> + </entry> + <entry> + <key>Log Payload</key> + <value> + <name>Log Payload</name> + </value> + </entry> + <entry> + <key>Attributes to Log</key> + <value> + <name>Attributes to Log</name> + </value> + </entry> + <entry> + <key>attributes-to-log-regex</key> + <value> + <name>attributes-to-log-regex</name> + </value> + </entry> + <entry> + <key>Attributes to Ignore</key> + <value> + <name>Attributes to Ignore</name> + </value> + </entry> + <entry> + <key>attributes-to-ignore-regex</key> + <value> + <name>attributes-to-ignore-regex</name> + </value> + </entry> + <entry> + <key>Log prefix</key> + <value> + <name>Log prefix</name> + </value> + </entry> + <entry> + <key>character-set</key> + <value> + <name>character-set</name> + </value> + </entry> + </descriptors> + <executionNode>ALL</executionNode> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>Log Level</key> + <value>info</value> + </entry> + <entry> + <key>Log Payload</key> + <value>true</value> + </entry> + <entry> + <key>Attributes to Log</key> + </entry> + <entry> + <key>attributes-to-log-regex</key> + <value>.*</value> + </entry> + <entry> + <key>Attributes to Ignore</key> + </entry> + <entry> + <key>attributes-to-ignore-regex</key> + </entry> + <entry> + <key>Log prefix</key> + </entry> + <entry> + <key>character-set</key> + <value>UTF-8</value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>0 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>LogAttribute</name> + <relationships> + <autoTerminate>true</autoTerminate> + <name>success</name> + </relationships> + <state>STOPPED</state> + <style></style> + <type>org.apache.nifi.processors.standard.LogAttribute</type> + </processors> + <processors> + <id>ff49910d-06bb-37ee-0000-000000000000</id> + <parentGroupId>21ae0bd6-5db6-3a47-0000-000000000000</parentGroupId> + <position> + <x>1.1368683772161603E-13</x> + <y>0.0</y> + </position> + <bundle> + <artifact>nifi-standard-nar</artifact> + <group>org.apache.nifi</group> + <version>1.4.0-SNAPSHOT</version> + </bundle> + <config> + <bulletinLevel>WARN</bulletinLevel> + <comments></comments> + <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount> + <descriptors> + <entry> + <key>File Size</key> + <value> + <name>File Size</name> + </value> + </entry> + <entry> + <key>Batch Size</key> + <value> + <name>Batch Size</name> + </value> + </entry> + <entry> + <key>Data Format</key> + <value> + <name>Data Format</name> + </value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value> + <name>Unique FlowFiles</name> + </value> + </entry> + <entry> + <key>generate-ff-custom-text</key> + <value> + <name>generate-ff-custom-text</name> + </value> + </entry> + </descriptors> + <executionNode>ALL</executionNode> + <lossTolerant>false</lossTolerant> + <penaltyDuration>30 sec</penaltyDuration> + <properties> + <entry> + <key>File Size</key> + <value>0B</value> + </entry> + <entry> + <key>Batch Size</key> + <value>1</value> + </entry> + <entry> + <key>Data Format</key> + <value>Text</value> + </entry> + <entry> + <key>Unique FlowFiles</key> + <value>false</value> + </entry> + <entry> + <key>generate-ff-custom-text</key> + <value>This is a plaintext message. </value> + </entry> + </properties> + <runDurationMillis>0</runDurationMillis> + <schedulingPeriod>1 sec</schedulingPeriod> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <yieldDuration>1 sec</yieldDuration> + </config> + <name>GenerateFlowFile</name> + <relationships> + <autoTerminate>false</autoTerminate> + <name>success</name> + </relationships> + <state>STOPPED</state> + <style></style> + <type>org.apache.nifi.processors.standard.GenerateFlowFile</type> + </processors> + </snippet> + <timestamp>09/05/2017 14:51:01 PDT</timestamp> +</template>
