http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties new file mode 100644 index 0000000..07c5ba7 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_protected_unknown.properties @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=n2z+tTTbHuZ4V4V2||uWhdasyDXD4ZG2lMAes/vqh6u4vaz4xgL4aEbF4Y/dXevqk3ulRcOwf1vc4RDQ== +nifi.sensitive.props.key.protected=unknown +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=oBjT92hIGRElIGOh||MZ6uYuWNBrOA6usq/Jt3DaD2e4otNirZDytac/w/KFe0HOkrJR03vcbo +nifi.security.keystorePasswd.protected=unknown +nifi.security.keyPasswd=ac/BaE35SL/esLiJ||+ULRvRLYdIDA2VqpE0eQXDEMjaLBMG2kbKOdOwBk/hGebDKlVg== +nifi.security.keyPasswd.protected=unknown +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec
http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties new file mode 100644 index 0000000..b81c369 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected.properties @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=thisIsABadKeystorePassword +nifi.security.keyPasswd=thisIsABadKeyPassword +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties new file mode 100644 index 0000000..4ad5784 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/resources/conf/nifi_with_sensitive_properties_unprotected_extra_line.properties @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=thisIsABadSensitiveKeyPassword +nifi.sensitive.props.key.protected= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text, nifi.version + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=thisIsABadKeystorePassword +nifi.security.keyPasswd=thisIsABadKeyPassword +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index e367e2a..5938fb2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -81,8 +81,11 @@ <nifi.nar.working.directory>./work/nar/</nifi.nar.working.directory> <nifi.documentation.working.directory>./work/docs/components</nifi.documentation.working.directory> + <nifi.sensitive.props.key.protected /> <nifi.sensitive.props.algorithm>PBEWITHMD5AND256BITAES-CBC-OPENSSL</nifi.sensitive.props.algorithm> <nifi.sensitive.props.provider>BC</nifi.sensitive.props.provider> + <nifi.sensitive.props.additional.keys /> + <nifi.h2.url.append>;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE</nifi.h2.url.append> <nifi.remote.input.socket.port>9990</nifi.remote.input.socket.port> http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf index 55679c2..f4a6c29 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf @@ -51,6 +51,8 @@ java.arg.13=-XX:+UseG1GC #Set headless mode by default java.arg.14=-Djava.awt.headless=true +#Set sensitive properties key +nifi.bootstrap.sensitive.key= ### # Notification Services for notifying interested parties when NiFi is stopped, started, dies http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties index 485b60e..bc5611e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties @@ -131,8 +131,10 @@ nifi.web.jetty.threads=${nifi.web.jetty.threads} # security properties # nifi.sensitive.props.key= +nifi.sensitive.props.key.protected=${nifi.sensitive.props.key.protected} nifi.sensitive.props.algorithm=${nifi.sensitive.props.algorithm} nifi.sensitive.props.provider=${nifi.sensitive.props.provider} +nifi.sensitive.props.additional.keys=${nifi.sensitive.props.additional.keys} nifi.security.keystore=${nifi.security.keystore} nifi.security.keystoreType=${nifi.security.keystoreType} http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml index 8f4f954..830b83e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/pom.xml @@ -34,6 +34,11 @@ </dependency> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-properties-loader</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-documentation</artifactId> <scope>compile</scope> </dependency> @@ -41,5 +46,10 @@ <groupId>org.slf4j</groupId> <artifactId>jul-to-slf4j</artifactId> </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <scope>test</scope> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java index 6d0fa97..44529d2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/main/java/org/apache/nifi/NiFi.java @@ -21,6 +21,9 @@ import java.io.IOException; import java.lang.Thread.UncaughtExceptionHandler; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Timer; import java.util.TimerTask; import java.util.concurrent.Executors; @@ -30,12 +33,13 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; - import org.apache.nifi.documentation.DocGenerator; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.ExtensionMapping; import org.apache.nifi.nar.NarClassLoaders; import org.apache.nifi.nar.NarUnpacker; +import org.apache.nifi.properties.NiFiPropertiesLoader; +import org.apache.nifi.properties.SensitivePropertyProtectionException; import org.apache.nifi.util.FileUtils; import org.apache.nifi.util.NiFiProperties; import org.slf4j.Logger; @@ -45,6 +49,7 @@ import org.slf4j.bridge.SLF4JBridgeHandler; public class NiFi { private static final Logger logger = LoggerFactory.getLogger(NiFi.class); + private static final String KEY_FLAG = "-k"; private final NiFiServer nifiServer; private final BootstrapListener bootstrapListener; @@ -158,7 +163,7 @@ public class NiFi { } logger.info("Jetty web server shutdown completed (nicely or otherwise)."); } catch (final Throwable t) { - logger.warn("Problem occured ensuring Jetty web server was properly terminated due to " + t); + logger.warn("Problem occurred ensuring Jetty web server was properly terminated due to " + t); } } @@ -183,14 +188,14 @@ public class NiFi { }); final AtomicInteger occurrencesOutOfRange = new AtomicInteger(0); - final AtomicInteger occurences = new AtomicInteger(0); + final AtomicInteger occurrences = new AtomicInteger(0); final Runnable command = new Runnable() { @Override public void run() { final long curMillis = System.currentTimeMillis(); final long difference = curMillis - lastTriggerMillis.get(); final long millisOff = Math.abs(difference - 2000L); - occurences.incrementAndGet(); + occurrences.incrementAndGet(); if (millisOff > 500L) { occurrencesOutOfRange.incrementAndGet(); } @@ -206,7 +211,7 @@ public class NiFi { future.cancel(true); service.shutdownNow(); - if (occurences.get() < minRequiredOccurrences || occurrencesOutOfRange.get() > maxOccurrencesOutOfRange) { + if (occurrences.get() < minRequiredOccurrences || occurrencesOutOfRange.get() > maxOccurrencesOutOfRange) { logger.warn("NiFi has detected that this box is not responding within the expected timing interval, which may cause " + "Processors to be scheduled erratically. Please see the NiFi documentation for more information."); } @@ -224,9 +229,96 @@ public class NiFi { public static void main(String[] args) { logger.info("Launching NiFi..."); try { - new NiFi(NiFiProperties.createBasicNiFiProperties(null, null)); + NiFiProperties properties = initializeProperties(args); + new NiFi(properties); } catch (final Throwable t) { logger.error("Failure to launch NiFi due to " + t, t); } } + + private static NiFiProperties initializeProperties(String[] args) { + // Try to get key + // If key doesn't exist, instantiate without + // Load properties + // If properties are protected and key missing, throw RuntimeException + + try { + String key = loadFormattedKey(args); + // The key might be empty or null when it is passed to the loader + try { + NiFiProperties properties = NiFiPropertiesLoader.withKey(key).get(); + logger.info("Loaded {} properties", properties.size()); + return properties; + } catch (SensitivePropertyProtectionException e) { + final String msg = "There was an issue decrypting protected properties"; + logger.error(msg, e); + throw new IllegalArgumentException(msg); + } + } catch (IllegalArgumentException e) { + final String msg = "The bootstrap process did not provide a valid key and there are protected properties present in the properties file"; + logger.error(msg, e); + throw new IllegalArgumentException(msg); + } + } + + private static String loadFormattedKey(String[] args) { + String key = null; + List<String> parsedArgs = parseArgs(args); + // Check if args contain protection key + if (parsedArgs.contains(KEY_FLAG)) { + key = getKeyFromArgs(parsedArgs); + + // Format the key (check hex validity and remove spaces) + key = formatHexKey(key); + if (!isHexKeyValid(key)) { + throw new IllegalArgumentException("The key was not provided in valid hex format and of the correct length"); + } + + return key; + } else { + // throw new IllegalStateException("No key provided from bootstrap"); + return ""; + } + } + + private static String getKeyFromArgs(List<String> parsedArgs) { + String key; + logger.debug("The bootstrap process provided the " + KEY_FLAG + " flag"); + int i = parsedArgs.indexOf(KEY_FLAG); + if (parsedArgs.size() <= i + 1) { + logger.error("The bootstrap process passed the {} flag without a key", KEY_FLAG); + throw new IllegalArgumentException("The bootstrap process provided the " + KEY_FLAG + " flag but no key"); + } + key = parsedArgs.get(i + 1); + logger.info("Read property protection key from bootstrap process"); + return key; + } + + private static List<String> parseArgs(String[] args) { + List<String> parsedArgs = new ArrayList<>(Arrays.asList(args)); + for (int i = 0; i < parsedArgs.size(); i++) { + if (parsedArgs.get(i).startsWith(KEY_FLAG + " ")) { + String[] split = parsedArgs.get(i).split(" ", 2); + parsedArgs.set(i, split[0]); + parsedArgs.add(i + 1, split[1]); + break; + } + } + return parsedArgs; + } + + private static String formatHexKey(String input) { + if (input == null || input.trim().isEmpty()) { + return ""; + } + return input.replaceAll("[^0-9a-fA-F]", "").toLowerCase(); + } + + private static boolean isHexKeyValid(String key) { + if (key == null || key.trim().isEmpty()) { + return false; + } + // Key length is in "nibbles" (i.e. one hex char = 4 bits) + return Arrays.asList(128, 196, 256).contains(key.length() * 4) && key.matches("^[0-9a-fA-F]*$"); + } } http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy new file mode 100644 index 0000000..f192d79 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/groovy/org/apache/nifi/NiFiGroovyTest.groovy @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi + +import ch.qos.logback.classic.spi.LoggingEvent +import ch.qos.logback.core.AppenderBase +import org.apache.nifi.properties.AESSensitivePropertyProvider +import org.apache.nifi.properties.NiFiPropertiesLoader +import org.apache.nifi.properties.StandardNiFiProperties +import org.apache.nifi.util.NiFiProperties +import org.bouncycastle.jce.provider.BouncyCastleProvider +import org.junit.After +import org.junit.AfterClass +import org.junit.Before +import org.junit.BeforeClass +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.JUnit4 +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.slf4j.bridge.SLF4JBridgeHandler + +import java.security.Security + +@RunWith(JUnit4.class) +class NiFiGroovyTest extends GroovyTestCase { + private static final Logger logger = LoggerFactory.getLogger(NiFiGroovyTest.class) + + private static String originalPropertiesPath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH) + + private static final String TEST_RES_PATH = NiFiGroovyTest.getClassLoader().getResource(".").toURI().getPath() + private static final File workDir = new File("./target/work/jetty/") + + @BeforeClass + public static void setUpOnce() throws Exception { + Security.addProvider(new BouncyCastleProvider()) + + SLF4JBridgeHandler.install() + + logger.metaClass.methodMissing = { String name, args -> + logger.info("[${name?.toUpperCase()}] ${(args as List).join(" ")}") + } + + logger.info("Identified test resources path as ${TEST_RES_PATH}") + } + + @Before + public void setUp() throws Exception { + if (!workDir.exists()) { + workDir.mkdirs() + } + } + + @After + public void tearDown() throws Exception { + NiFiPropertiesLoader.@sensitivePropertyProviderFactory = null + TestAppender.reset() + System.setIn(System.in) + } + + @AfterClass + public static void tearDownOnce() { + if (originalPropertiesPath) { + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, originalPropertiesPath) + } + } + + @Test + public void testInitializePropertiesShouldHandleNoBootstrapKey() throws Exception { + // Arrange + def args = [] as String[] + + String plainPropertiesPath = "${TEST_RES_PATH}/NiFiProperties/conf/nifi.properties" + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, plainPropertiesPath) + + // Act + NiFiProperties loadedProperties = NiFi.initializeProperties(args) + + // Assert + assert loadedProperties.size() > 0 + } + + @Test + public void testMainShouldHandleNoBootstrapKeyWithProtectedProperties() throws Exception { + // Arrange + def args = [] as String[] + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties") + + // Act + NiFi.main(args) + + // Assert + assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file" + } + + @Test + public void testParseArgsShouldSplitCombinedArgs() throws Exception { + // Arrange + final String DIFFERENT_KEY = "0" * 64 + def args = ["-k ${DIFFERENT_KEY}"] as String[] + + // Act + def parsedArgs = NiFi.parseArgs(args) + + // Assert + assert parsedArgs.size() == 2 + assert parsedArgs == args.join(" ").split(" ") as List + } + + @Test + public void testMainShouldHandleBadArgs() throws Exception { + // Arrange + def args = ["-k"] as String[] + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties") + + // Act + NiFi.main(args) + + // Assert + assert TestAppender.events.collect { + it.getFormattedMessage() + }.contains("The bootstrap process passed the -k flag without a key") + assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file" + } + + @Test + public void testMainShouldHandleMalformedBootstrapKey() throws Exception { + // Arrange + def args = ["-k", "BAD KEY"] as String[] + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties") + + // Act + NiFi.main(args) + + // Assert + assert TestAppender.events.last().getMessage() == "Failure to launch NiFi due to java.lang.IllegalArgumentException: The bootstrap process did not provide a valid key and there are protected properties present in the properties file" + } + + @Test + public void testInitializePropertiesShouldSetBootstrapKeyFromArgs() throws Exception { + // Arrange + final String DIFFERENT_KEY = "0" * 64 + def args = ["-k", DIFFERENT_KEY] as String[] + + String testPropertiesPath = "${TEST_RES_PATH}/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties" + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, testPropertiesPath) + + NiFiProperties unprocessedProperties = new NiFiPropertiesLoader().loadRaw(new File(testPropertiesPath)) + def protectedKeys = getProtectedKeys(unprocessedProperties) + logger.info("Reading from raw properties file gives protected properties: ${protectedKeys}") + + // Act + NiFiProperties properties = NiFi.initializeProperties(args) + + // Assert + + // Ensure that there were protected properties, they were encrypted using AES/GCM (128/256 bit key), and they were decrypted (raw value != retrieved value) + assert !hasProtectedKeys(properties) + def unprotectedProperties = decrypt(unprocessedProperties, DIFFERENT_KEY) + getProtectedPropertyKeys(unprocessedProperties).every { k, v -> + String rawValue = unprocessedProperties.getProperty(k) + logger.raw("${k} -> ${rawValue}") + String retrievedValue = properties.getProperty(k) + logger.decrypted("${k} -> ${retrievedValue}") + + assert v =~ "aes/gcm" + + logger.assert("${retrievedValue} != ${rawValue}") + assert retrievedValue != rawValue + + String decryptedProperty = unprotectedProperties.getProperty(k) + logger.assert("${retrievedValue} == ${decryptedProperty}") + assert retrievedValue == decryptedProperty + } + } + + private static boolean hasProtectedKeys(NiFiProperties properties) { + properties.getPropertyKeys().any { it.endsWith(".protected") } + } + + private static Map<String, String> getProtectedPropertyKeys(NiFiProperties properties) { + getProtectedKeys(properties).collectEntries { String key -> + [(key): properties.getProperty(key + ".protected")] + } + } + + private static Set<String> getProtectedKeys(NiFiProperties properties) { + properties.getPropertyKeys().findAll { it.endsWith(".protected") } + } + + private static NiFiProperties decrypt(NiFiProperties encryptedProperties, String keyHex) { + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(keyHex) + def map = encryptedProperties.getPropertyKeys().collectEntries { String key -> + if (encryptedProperties.getProperty(key + ".protected") == spp.getIdentifierKey()) { + [(key): spp.unprotect(encryptedProperties.getProperty(key))] + } else if (!key.endsWith(".protected")) { + [(key): encryptedProperties.getProperty(key)] + } + } + new StandardNiFiProperties(map as Properties) + } + + @Test + public void testShouldValidateKeys() { + // Arrange + final List<String> VALID_KEYS = [ + "0" * 64, // 256 bit keys + "ABCDEF01" * 8, + "0123" * 8, // 128 bit keys + "0123456789ABCDEFFEDCBA9876543210", + "0123456789ABCDEFFEDCBA9876543210".toLowerCase(), + ] + + // Act + def isValid = VALID_KEYS.collectEntries { String key -> [(key): NiFi.isHexKeyValid(key)] } + logger.info("Key validity: ${isValid}") + + // Assert + assert isValid.every { k, v -> v } + } + + @Test + public void testShouldNotValidateInvalidKeys() { + // Arrange + final List<String> VALID_KEYS = [ + "0" * 63, + "ABCDEFG1" * 8, + "0123" * 9, + "0123456789ABCDEFFEDCBA987654321", + "0123456789ABCDEF FEDCBA9876543210".toLowerCase(), + null, + "", + " " + ] + + // Act + def isValid = VALID_KEYS.collectEntries { String key -> [(key): NiFi.isHexKeyValid(key)] } + logger.info("Key validity: ${isValid}") + + // Assert + assert isValid.every { k, v -> !v } + } +} + +public class TestAppender extends AppenderBase<LoggingEvent> { + static List<LoggingEvent> events = new ArrayList<>(); + + @Override + protected void append(LoggingEvent e) { + synchronized (events) { + events.add(e); + } + } + + public static void reset() { + synchronized (events) { + events.clear(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties new file mode 100644 index 0000000..f341c01 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi.properties @@ -0,0 +1,188 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=1.0.0-SNAPSHOT +nifi.flow.configuration.file=./target/conf/flow.xml.gz +nifi.flow.configuration.archive.enabled=true +nifi.flow.configuration.archive.dir=./target/conf/archive/ +nifi.flow.configuration.archive.max.time=30 days +nifi.flow.configuration.archive.max.storage=500 MB +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +# If a component has no work to do (is "bored"), how long should we wait before checking again for work? +nifi.bored.yield.duration=10 millis + +nifi.authorizer.configuration.file=./target/conf/authorizers.xml +nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml +nifi.templates.directory=./target/conf/templates +nifi.ui.banner.text= +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/lib +nifi.nar.working.directory=./target/work/nar/ +nifi.documentation.working.directory=./target/work/docs/components + +#################### +# State Management # +#################### +nifi.state.management.configuration.file=./target/conf/state-management.xml +# The ID of the local state provider +nifi.state.management.provider.local=local-provider +# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster. +nifi.state.management.provider.cluster=zk-provider +# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server +nifi.state.management.embedded.zookeeper.start=false +# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true +nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties + + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.directory=./target/flowfile_repository +nifi.flowfile.repository.partitions=256 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.flowfile.repository.always.sync=false + +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository +nifi.content.repository.archive.max.retention.period=12 hours +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=true +nifi.content.repository.always.sync=false +nifi.content.viewer.url=/nifi-content-viewer/ + +# Provenance Repository Properties +nifi.provenance.repository.implementation=org.apache.nifi.provenance.PersistentProvenanceRepository + +# Persistent Provenance Repository Properties +nifi.provenance.repository.directory.default=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB +nifi.provenance.repository.query.threads=2 +nifi.provenance.repository.index.threads=1 +nifi.provenance.repository.compress.on.rollover=true +nifi.provenance.repository.always.sync=false +nifi.provenance.repository.journal.count=16 +# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: +# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details +nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship +# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type +nifi.provenance.repository.indexed.attributes= +# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository +# but should provide better performance +nifi.provenance.repository.index.shard.size=500 MB +# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from +# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. +nifi.provenance.repository.max.attribute.length=65536 + +# Volatile Provenance Respository Properties +nifi.provenance.repository.buffer.size=100000 + +# Component Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.repository.buffer.size=1440 +nifi.components.status.snapshot.frequency=1 min + +# Site to Site properties +nifi.remote.input.host= +nifi.remote.input.secure=false +nifi.remote.input.socket.port= +nifi.remote.input.http.enabled=true +nifi.remote.input.http.transaction.ttl=30 sec + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty +nifi.web.jetty.threads=200 + +# security properties # +nifi.sensitive.props.key= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer=file-provider +nifi.security.user.login.identity.provider= +nifi.security.ocsp.responder.url= +nifi.security.ocsp.responder.certificate= + +# Identity Mapping Properties # +# These properties allow normalizing user identities such that identities coming from different identity providers +# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing +# DNs from certificates and principals from Kerberos into a common identity string: +# +# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$ +# nifi.security.identity.mapping.value.dn=$1@$2 +# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$ +# nifi.security.identity.mapping.value.kerb=$1@$2 + +# cluster common properties (all nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=10 +nifi.cluster.node.event.history.size=25 +nifi.cluster.node.connection.timeout=5 sec +nifi.cluster.node.read.timeout=5 sec +nifi.cluster.firewall.file= + +# How long a request should be allowed to hold a 'lock' on a component. # +nifi.cluster.request.replication.claim.timeout=15 secs + +# zookeeper properties, used for cluster management # +nifi.zookeeper.connect.string= +nifi.zookeeper.connect.timeout=3 secs +nifi.zookeeper.session.timeout=3 secs +nifi.zookeeper.root.node=/nifi + +# kerberos # +nifi.kerberos.krb5.file= +nifi.kerberos.service.principal= +nifi.kerberos.keytab.location= +nifi.kerberos.authentication.expiration=12 hours \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties new file mode 100644 index 0000000..cbc9382 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes.properties @@ -0,0 +1,189 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=1.0.0-SNAPSHOT +nifi.flow.configuration.file=./target/conf/flow.xml.gz +nifi.flow.configuration.archive.enabled=true +nifi.flow.configuration.archive.dir=./target/conf/archive/ +nifi.flow.configuration.archive.max.time=30 days +nifi.flow.configuration.archive.max.storage=500 MB +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +# If a component has no work to do (is "bored"), how long should we wait before checking again for work? +nifi.bored.yield.duration=10 millis + +nifi.authorizer.configuration.file=./target/conf/authorizers.xml +nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml +nifi.templates.directory=./target/conf/templates +nifi.ui.banner.text=n8hL1zgQcYpG70Vm||e1hYsrc7FLvi1E9LcHM1VYeN5atWJIGg/WCsyuxxNqN1lK1ASGEZR8040NFZNqwsnbx+ +nifi.ui.banner.text.protected=aes/gcm/256 +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/lib +nifi.nar.working.directory=./target/work/nar/ +nifi.documentation.working.directory=./target/work/docs/components + +#################### +# State Management # +#################### +nifi.state.management.configuration.file=./target/conf/state-management.xml +# The ID of the local state provider +nifi.state.management.provider.local=local-provider +# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster. +nifi.state.management.provider.cluster=zk-provider +# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server +nifi.state.management.embedded.zookeeper.start=false +# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true +nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties + + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.directory=./target/flowfile_repository +nifi.flowfile.repository.partitions=256 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.flowfile.repository.always.sync=false + +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository +nifi.content.repository.archive.max.retention.period=12 hours +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=true +nifi.content.repository.always.sync=false +nifi.content.viewer.url=/nifi-content-viewer/ + +# Provenance Repository Properties +nifi.provenance.repository.implementation=org.apache.nifi.provenance.PersistentProvenanceRepository + +# Persistent Provenance Repository Properties +nifi.provenance.repository.directory.default=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB +nifi.provenance.repository.query.threads=2 +nifi.provenance.repository.index.threads=1 +nifi.provenance.repository.compress.on.rollover=true +nifi.provenance.repository.always.sync=false +nifi.provenance.repository.journal.count=16 +# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: +# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details +nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship +# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type +nifi.provenance.repository.indexed.attributes= +# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository +# but should provide better performance +nifi.provenance.repository.index.shard.size=500 MB +# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from +# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. +nifi.provenance.repository.max.attribute.length=65536 + +# Volatile Provenance Respository Properties +nifi.provenance.repository.buffer.size=100000 + +# Component Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.repository.buffer.size=1440 +nifi.components.status.snapshot.frequency=1 min + +# Site to Site properties +nifi.remote.input.host= +nifi.remote.input.secure=false +nifi.remote.input.socket.port= +nifi.remote.input.http.enabled=true +nifi.remote.input.http.transaction.ttl=30 sec + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty +nifi.web.jetty.threads=200 + +# security properties # +nifi.sensitive.props.key= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys=nifi.ui.banner.text + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer=file-provider +nifi.security.user.login.identity.provider= +nifi.security.ocsp.responder.url= +nifi.security.ocsp.responder.certificate= + +# Identity Mapping Properties # +# These properties allow normalizing user identities such that identities coming from different identity providers +# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing +# DNs from certificates and principals from Kerberos into a common identity string: +# +# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$ +# nifi.security.identity.mapping.value.dn=$1@$2 +# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$ +# nifi.security.identity.mapping.value.kerb=$1@$2 + +# cluster common properties (all nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=10 +nifi.cluster.node.event.history.size=25 +nifi.cluster.node.connection.timeout=5 sec +nifi.cluster.node.read.timeout=5 sec +nifi.cluster.firewall.file= + +# How long a request should be allowed to hold a 'lock' on a component. # +nifi.cluster.request.replication.claim.timeout=15 secs + +# zookeeper properties, used for cluster management # +nifi.zookeeper.connect.string= +nifi.zookeeper.connect.timeout=3 secs +nifi.zookeeper.session.timeout=3 secs +nifi.zookeeper.root.node=/nifi + +# kerberos # +nifi.kerberos.krb5.file= +nifi.kerberos.service.principal= +nifi.kerberos.keytab.location= +nifi.kerberos.authentication.expiration=12 hours \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties new file mode 100644 index 0000000..f170d7e --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/NiFiProperties/conf/nifi_with_sensitive_properties_protected_aes_different_key.properties @@ -0,0 +1,192 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=1.0.0-SNAPSHOT +nifi.flow.configuration.file=./target/conf/flow.xml.gz +nifi.flow.configuration.archive.enabled=true +nifi.flow.configuration.archive.dir=./target/conf/archive/ +nifi.flow.configuration.archive.max.time=30 days +nifi.flow.configuration.archive.max.storage=500 MB +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=500 ms +nifi.administrative.yield.duration=30 sec +# If a component has no work to do (is "bored"), how long should we wait before checking again for work? +nifi.bored.yield.duration=10 millis + +nifi.authorizer.configuration.file=./target/conf/authorizers.xml +nifi.login.identity.provider.configuration.file=./target/conf/login-identity-providers.xml +nifi.templates.directory=./target/conf/templates +nifi.ui.banner.text=dXwnu9mLyPETJrq1||n9e5dk5+HSTBCGOA/Sy6VYzwPw3baeRNvglalA1Pr1PcToyc4/qT6md24YOP4xVz14jd +nifi.ui.banner.text.protected=aes/gcm/256 +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/lib +nifi.nar.working.directory=./target/work/nar/ +nifi.documentation.working.directory=./target/work/docs/components + +#################### +# State Management # +#################### +nifi.state.management.configuration.file=./target/conf/state-management.xml +# The ID of the local state provider +nifi.state.management.provider.local=local-provider +# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster. +nifi.state.management.provider.cluster=zk-provider +# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server +nifi.state.management.embedded.zookeeper.start=false +# Properties file that provides the ZooKeeper properties to use if <nifi.state.management.embedded.zookeeper.start> is set to true +nifi.state.management.embedded.zookeeper.properties=./target/conf/zookeeper.properties + + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository +nifi.flowfile.repository.directory=./target/flowfile_repository +nifi.flowfile.repository.partitions=256 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.flowfile.repository.always.sync=false + +nifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager +nifi.queue.swap.threshold=20000 +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository +nifi.content.repository.archive.max.retention.period=12 hours +nifi.content.repository.archive.max.usage.percentage=50% +nifi.content.repository.archive.enabled=true +nifi.content.repository.always.sync=false +nifi.content.viewer.url=/nifi-content-viewer/ + +# Provenance Repository Properties +nifi.provenance.repository.implementation=org.apache.nifi.provenance.PersistentProvenanceRepository + +# Persistent Provenance Repository Properties +nifi.provenance.repository.directory.default=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB +nifi.provenance.repository.query.threads=2 +nifi.provenance.repository.index.threads=1 +nifi.provenance.repository.compress.on.rollover=true +nifi.provenance.repository.always.sync=false +nifi.provenance.repository.journal.count=16 +# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: +# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details +nifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship +# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type +nifi.provenance.repository.indexed.attributes= +# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository +# but should provide better performance +nifi.provenance.repository.index.shard.size=500 MB +# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from +# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved. +nifi.provenance.repository.max.attribute.length=65536 + +# Volatile Provenance Respository Properties +nifi.provenance.repository.buffer.size=100000 + +# Component Status Repository +nifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository +nifi.components.status.repository.buffer.size=1440 +nifi.components.status.snapshot.frequency=1 min + +# Site to Site properties +nifi.remote.input.host= +nifi.remote.input.secure=false +nifi.remote.input.socket.port= +nifi.remote.input.http.enabled=true +nifi.remote.input.http.transaction.ttl=30 sec + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty +nifi.web.jetty.threads=200 + +# security properties # +nifi.sensitive.props.key=dQU402Mz4J+t+e18||6+ictR0Nssq3/rR/d8fq5CFAKmpakr9jCyPIJYxG7n6D86gxsu2TRp4M48ugUw== +nifi.sensitive.props.key.protected=aes/gcm/256 +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=Q8T3wv+Xl2ie98GV||qsuY9wa/Rt27cqFXs8ebX25E1iSbFAEFcD0cjCwrl3Tw6HghQjBIaCzQ +nifi.security.keystorePasswd.protected=aes/gcm/256 +nifi.security.keyPasswd=1S0XmoiAr379B8rg||PPZzjdw9BAJSon9g4xm9uscFhCCyk734FTjXtRnBXUy819zsoQ== +nifi.security.keyPasswd.protected=aes/gcm/256 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer=file-provider +nifi.security.user.login.identity.provider= +nifi.security.ocsp.responder.url= +nifi.security.ocsp.responder.certificate= + +# Identity Mapping Properties # +# These properties allow normalizing user identities such that identities coming from different identity providers +# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing +# DNs from certificates and principals from Kerberos into a common identity string: +# +# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$ +# nifi.security.identity.mapping.value.dn=$1@$2 +# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$ +# nifi.security.identity.mapping.value.kerb=$1@$2 + +# cluster common properties (all nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=10 +nifi.cluster.node.event.history.size=25 +nifi.cluster.node.connection.timeout=5 sec +nifi.cluster.node.read.timeout=5 sec +nifi.cluster.firewall.file= + +# How long a request should be allowed to hold a 'lock' on a component. # +nifi.cluster.request.replication.claim.timeout=15 secs + +# zookeeper properties, used for cluster management # +nifi.zookeeper.connect.string= +nifi.zookeeper.connect.timeout=3 secs +nifi.zookeeper.session.timeout=3 secs +nifi.zookeeper.root.node=/nifi + +# kerberos # +nifi.kerberos.krb5.file= +nifi.kerberos.service.principal= +nifi.kerberos.keytab.location= +nifi.kerberos.authentication.expiration=12 hours \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/logback-test.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/logback-test.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/logback-test.xml new file mode 100644 index 0000000..7f0a03b --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-runtime/src/test/resources/logback-test.xml @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender"> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%-4r [%t] %-5p %c - %m%n</pattern> + </encoder> + </appender> + <appender name="TEST" class="org.apache.nifi.TestAppender"> + <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> + <pattern>%-4r [%t] %-5p %c - %m%n</pattern> + </encoder> + </appender> + + <logger name="org.apache.nifi.processor" level="DEBUG"/> + <root level="DEBUG"> + <appender-ref ref="CONSOLE"/> + <appender-ref ref="TEST"/> + </root> +</configuration> http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/http/StandardHttpFlowFileServerProtocol.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/http/StandardHttpFlowFileServerProtocol.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/http/StandardHttpFlowFileServerProtocol.java index e71ac1d..0795b5d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/http/StandardHttpFlowFileServerProtocol.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/http/StandardHttpFlowFileServerProtocol.java @@ -93,7 +93,7 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr HttpServerCommunicationsSession commSession = (HttpServerCommunicationsSession) commsSession; commSession.setResponseCode(response); - if (isTransfer) { + if(isTransfer){ switch (response) { case NO_MORE_DATA: logger.debug("{} There's no data to send.", this); @@ -138,8 +138,8 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr ByteArrayOutputStream bos = new ByteArrayOutputStream(); Transaction.TransactionState currentStatus = commSession.getStatus(); - if (isTransfer) { - switch (currentStatus) { + if(isTransfer){ + switch (currentStatus){ case DATA_EXCHANGED: String clientChecksum = commSession.getChecksum(); logger.debug("readTransactionResponse. clientChecksum={}", clientChecksum); @@ -151,7 +151,7 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr break; } } else { - switch (currentStatus) { + switch (currentStatus){ case TRANSACTION_STARTED: logger.debug("readTransactionResponse. returning CONTINUE_TRANSACTION."); // We don't know if there's more data to receive, so just continue it. @@ -161,7 +161,7 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr // Checksum was successfully validated at client side, or BAD_CHECKSUM is returned. ResponseCode responseCode = commSession.getResponseCode(); logger.debug("readTransactionResponse. responseCode={}", responseCode); - if (responseCode.containsMessage()) { + if(responseCode.containsMessage()){ responseCode.writeResponse(new DataOutputStream(bos), ""); } else { responseCode.writeResponse(new DataOutputStream(bos)); @@ -228,8 +228,8 @@ public class StandardHttpFlowFileServerProtocol extends AbstractFlowFileServerPr } @Override - public void sendPeerList(Peer peer, Optional<ClusterNodeInformation> clusterNodeInfo, String remoteInputHost, - int remoteInputPort, int remoteInputHttpPort, boolean isSiteToSiteSecure) throws IOException { + public void sendPeerList(Peer peer, Optional<ClusterNodeInformation> clusterNodeInfo, String remoteInputHost, int remoteInputPort, int remoteInputHttpPort, + boolean isSiteToSiteSecure) throws IOException { } @Override http://git-wip-us.apache.org/repos/asf/nifi/blob/c638191a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/socket/SocketFlowFileServerProtocol.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/socket/SocketFlowFileServerProtocol.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/socket/SocketFlowFileServerProtocol.java index e965cf4..8ecf96c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/socket/SocketFlowFileServerProtocol.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-site-to-site/src/main/java/org/apache/nifi/remote/protocol/socket/SocketFlowFileServerProtocol.java @@ -16,6 +16,16 @@ */ package org.apache.nifi.remote.protocol.socket; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.apache.nifi.remote.Peer; import org.apache.nifi.remote.RemoteResourceFactory; import org.apache.nifi.remote.StandardVersionNegotiator; @@ -31,17 +41,6 @@ import org.apache.nifi.remote.protocol.HandshakeProperties; import org.apache.nifi.remote.protocol.RequestType; import org.apache.nifi.remote.protocol.ResponseCode; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol { public static final String RESOURCE_NAME = "SocketFlowFileProtocol"; @@ -83,7 +82,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol validateHandshakeRequest(confirmed, peer, properties); } catch (HandshakeException e) { ResponseCode handshakeResult = e.getResponseCode(); - if (handshakeResult.containsMessage()) { + if(handshakeResult.containsMessage()){ handshakeResult.writeResponse(dos, e.getMessage()); } else { handshakeResult.writeResponse(dos); @@ -135,6 +134,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol } } + @Override public RequestType getRequestType(final Peer peer) throws IOException { if (!handshakeCompleted) { @@ -181,7 +181,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol nodeInfos = new ArrayList<>(clusterNodeInfo.get().getNodeInformation()); } else { final NodeInformation self = new NodeInformation(remoteInputHostVal, remoteInputPort, remoteInputHttpPort, remoteInputHttpPort, - isSiteToSiteSecure, 0); + isSiteToSiteSecure, 0); nodeInfos = Collections.singletonList(self); } @@ -214,6 +214,7 @@ public class SocketFlowFileServerProtocol extends AbstractFlowFileServerProtocol return RESOURCE_NAME; } + @Override public VersionNegotiator getVersionNegotiator() { return versionNegotiator;
