NIFI-292: Merging develop into NIFI-292
Project: http://git-wip-us.apache.org/repos/asf/incubator-nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-nifi/commit/6f5b6225 Tree: http://git-wip-us.apache.org/repos/asf/incubator-nifi/tree/6f5b6225 Diff: http://git-wip-us.apache.org/repos/asf/incubator-nifi/diff/6f5b6225 Branch: refs/heads/NIFI-292 Commit: 6f5b62253e182281462e514b430e06842c5eed0e Parents: 07f44d9 Author: Matt Gilman <[email protected]> Authored: Fri May 1 14:33:44 2015 -0400 Committer: Matt Gilman <[email protected]> Committed: Fri May 1 14:33:44 2015 -0400 ---------------------------------------------------------------------- README.md | 39 +- nifi-nar-maven-plugin/README.md | 18 +- nifi-parent/README.md | 16 +- nifi/README.md | 64 +- .../notification/OnPrimaryNodeStateChange.java | 44 ++ .../notification/PrimaryNodeState.java | 33 ++ nifi/nifi-assembly/NOTICE | 13 +- nifi/nifi-assembly/pom.xml | 29 +- .../org/apache/nifi/util/NiFiProperties.java | 111 +++- .../apache/nifi/util/NiFiPropertiesTest.java | 95 +++ .../NiFiProperties/conf/nifi.blank.properties | 128 ++++ .../NiFiProperties/conf/nifi.missing.properties | 126 ++++ .../NiFiProperties/conf/nifi.properties | 128 ++++ .../nifi/security/util/EncryptionMethod.java | 5 +- .../client/socket/EndpointConnectionPool.java | 2 +- .../src/main/asciidoc/administration-guide.adoc | 105 +++- .../images/nifi_first_launch_screenshot.png | Bin 0 -> 122540 bytes .../html/HtmlDocumentationWriter.java | 61 +- .../FullyDocumentedControllerService.java | 10 +- .../html/ProcessorDocumentationWriterTest.java | 6 +- .../impl/FileBasedClusterNodeFirewall.java | 18 +- .../impl/FileBasedClusterNodeFirewallTest.java | 69 ++- .../ClusterManagerProtocolSenderImplTest.java | 39 +- .../apache/nifi/controller/FlowController.java | 36 +- .../java/org/apache/nifi/nar/NarUnpacker.java | 131 +++-- .../org/apache/nifi/nar/NarUnpackerTest.java | 185 ++++++ .../resources/NarUnpacker/conf/nifi.properties | 129 +++++ .../resources/NarUnpacker/lib/dummy-one.nar | Bin 0 -> 1749 bytes .../NarUnpacker/lib/nifi-framework-nar.nar | Bin 0 -> 406 bytes .../resources/NarUnpacker/lib2/dummy-two.nar | Bin 0 -> 1751 bytes .../nifi/remote/StandardRemoteGroupPort.java | 40 +- .../org/apache/nifi/web/util/SnippetUtils.java | 2 +- .../nifi-hdfs-processors/pom.xml | 8 +- .../hadoop/AbstractHadoopProcessor.java | 51 +- .../hadoop/CreateHadoopSequenceFile.java | 2 +- .../nifi/processors/hadoop/FetchHDFS.java | 125 ++++ .../apache/nifi/processors/hadoop/GetHDFS.java | 174 +++--- .../processors/hadoop/GetHDFSSequenceFile.java | 12 +- .../apache/nifi/processors/hadoop/ListHDFS.java | 481 +++++++++++++++ .../apache/nifi/processors/hadoop/PutHDFS.java | 21 +- .../processors/hadoop/util/HDFSListing.java | 83 +++ .../nifi/processors/hadoop/util/LongSerDe.java | 48 ++ .../processors/hadoop/util/StringSerDe.java | 44 ++ .../org.apache.nifi.processor.Processor | 6 +- .../nifi/processors/hadoop/TestListHDFS.java | 357 ++++++++++++ .../nifi/provenance/IndexConfiguration.java | 16 +- .../PersistentProvenanceRepository.java | 219 ++++--- .../nifi/provenance/StandardRecordWriter.java | 20 +- .../nifi/provenance/lucene/LineageQuery.java | 103 ++-- .../provenance/serialization/RecordWriter.java | 3 +- .../nifi/provenance/toc/StandardTocReader.java | 51 +- .../nifi/provenance/toc/StandardTocWriter.java | 5 +- .../apache/nifi/provenance/toc/TocReader.java | 9 + .../apache/nifi/provenance/toc/TocWriter.java | 7 +- .../TestStandardRecordReaderWriter.java | 16 +- .../provenance/toc/TestStandardTocReader.java | 29 +- .../nifi-solr-bundle/nifi-solr-nar/pom.xml | 36 ++ .../src/main/resources/META-INF/LICENSE | 321 ++++++++++ .../src/main/resources/META-INF/NOTICE | 60 ++ .../nifi-solr-processors/pom.xml | 117 ++++ .../apache/nifi/processors/solr/GetSolr.java | 314 ++++++++++ .../processors/solr/PutSolrContentStream.java | 295 ++++++++++ .../nifi/processors/solr/SolrProcessor.java | 145 +++++ .../org.apache.nifi.processor.Processor | 16 + .../additionalDetails.html | 48 ++ .../solr/EmbeddedSolrServerFactory.java | 85 +++ .../nifi/processors/solr/TestGetSolr.java | 200 +++++++ .../solr/TestPutSolrContentStream.java | 416 +++++++++++++ .../src/test/resources/log4j.properties | 14 + .../src/test/resources/solr/solr.xml | 18 + .../testCollection/conf/_rest_managed.json | 3 + .../testCollection/conf/lang/stopwords_en.txt | 54 ++ .../resources/testCollection/conf/protwords.txt | 21 + .../resources/testCollection/conf/schema.xml | 21 + .../testCollection/conf/solrconfig.xml | 20 + .../resources/testCollection/conf/synonyms.txt | 29 + .../resources/testCollection/core.properties | 1 + .../testdata/test-csv-multiple-docs.csv | 2 + .../testdata/test-custom-json-single-doc.json | 15 + .../testdata/test-solr-json-multiple-docs.json | 18 + .../testdata/test-xml-multiple-docs.xml | 18 + nifi/nifi-nar-bundles/nifi-solr-bundle/pom.xml | 39 ++ .../nifi-standard-processors/pom.xml | 6 + .../processors/standard/DuplicateFlowFile.java | 81 +++ .../processors/standard/EncryptContent.java | 580 ++++++++++--------- .../processors/standard/ExecuteProcess.java | 301 +++++----- .../nifi/processors/standard/GetHTTP.java | 36 +- .../standard/util/OpenPGPKeyBasedEncryptor.java | 277 +++++++++ .../util/OpenPGPPasswordBasedEncryptor.java | 175 ++++++ .../standard/util/PasswordBasedEncryptor.java | 155 +++++ .../org.apache.nifi.processor.Processor | 1 + .../additionalDetails.html | 30 + .../standard/TestDetectDuplicate.java | 29 +- .../standard/TestDuplicateFlowFile.java | 36 ++ .../processors/standard/TestEncryptContent.java | 227 ++++++-- .../processors/standard/TestExecuteProcess.java | 82 +++ .../standard/TestHandleHttpRequest.java | 20 +- .../test/resources/TestEncryptContent/text.txt | 17 + .../resources/TestEncryptContent/text.txt.asc | 33 ++ .../cache/client/DistributedMapCacheClient.java | 23 +- .../DistributedMapCacheClientService.java | 75 ++- .../DistributedSetCacheClientService.java | 2 + .../additionalDetails.html | 45 -- .../distributed/cache/server/map/MapCache.java | 2 + .../cache/server/map/MapCacheServer.java | 117 ++-- .../cache/server/map/PersistentMapCache.java | 33 +- .../cache/server/map/SimpleMapCache.java | 22 + .../additionalDetails.html | 46 -- .../index.html | 67 --- .../additionalDetails.html | 49 -- nifi/nifi-nar-bundles/pom.xml | 1 + nifi/pom.xml | 33 +- 112 files changed, 7160 insertions(+), 1339 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 75b79c2..d7a04c4 100644 --- a/README.md +++ b/README.md @@ -24,9 +24,42 @@ Apache NiFi is an easy to use, powerful, and reliable system to process and dist ## Getting Started -- Build the nifi-nar-maven-plugin. Change directory to 'nifi-nar-maven-plugin' and -follow the directions found there. -- Build nifi. Change directory to 'nifi' and follow the directions found there. +- Read through the [quickstart guide for development](http://nifi.incubator.apache.org/development/quickstart.html). + It will include information on getting a local copy of the source, give pointers on issue + tracking, and provide some warnings about common problems with development environments. +- Optional: Build supporting modules. This should only be needed if the current 'nifi' module is in + the process of updating to a new version of either the 'nifi-parent' or 'nifi-nar-maven-plugin' + artifacts. + + If in doubt, just skip to building the main nifi project. If the build fails, come back here and + figure out which optional step you are missing; each entry below will give an example of the + errors you'll receive if that step needs to be followed. The version numbers may change but the + error text should still look very familiar. + - Install the nifi-parent pom. Change directory to 'nifi-parent' and follow the directions found + there in [README.md](nifi-parent/README.md). + + If you don't build the nifi-parent pom and the main nifi code relies on an unreleased version + you'll see an erorr like the following: + + [ERROR] Non-resolvable parent POM: Could not find artifact + org.apache.nifi:nifi-parent:pom:1.0.0-incubating-SNAPSHOT in example.snapshots.repo + (https://repository.example.com/content/repositories/snapshots) and + 'parent.relativePath' points at no local POM @ line 18, column 13 -> [Help 2] + - Build the nifi-nar-maven-plugin. Change directory to 'nifi-nar-maven-plugin' and + follow the directions found there in [README.md](nifi-nar-maven-plugin/README.md). + + If you don't build the nifi-nar-maven-plugin and the main nifi code relies on an unreleased + version you'll see an error like the following: + + [ERROR] Unresolveable build extension: Plugin + org.apache.nifi:nifi-nar-maven-plugin:1.0.1-incubating-SNAPSHOT or one of its + dependencies could not be resolved: Could not find artifact + org.apache.nifi:nifi-nar-maven-plugin:jar:1.0.1-incubating-SNAPSHOT -> [Help 2] +- Build nifi. Change directory to 'nifi' and follow the directions found there in + [README.md](nifi/README.md). +- Run NiFi. The directions found in the [README.md](nifi/README.md) file within the 'nifi' module + will also include how to run an instance of NiFi. For help on how to build your first data flow, + see the [NiFi User Guide](http://nifi.incubator.apache.org/docs/nifi-docs/user-guide.html). ## Documentation http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi-nar-maven-plugin/README.md ---------------------------------------------------------------------- diff --git a/nifi-nar-maven-plugin/README.md b/nifi-nar-maven-plugin/README.md index 1b554d9..4fd3a27 100644 --- a/nifi-nar-maven-plugin/README.md +++ b/nifi-nar-maven-plugin/README.md @@ -18,16 +18,25 @@ Apache NiFi NAR Maven Plugin helps to build NiFi Archive bundles to support the ## Table of Contents +- [Requirements](#requirements) - [Getting Started](#getting-started) - [Getting Help](#getting-help) -- [Requirements](#requirements) - [License](#license) - [Disclaimer](#disclaimer) +## Requirements +* JDK 1.7 or higher +* Apache Maven 3.0.5 or higher + ## Getting Started -To Build: -- Executed 'mvn clean install' +Building the nifi-nar-maven-plugin module should be rare since it will be released infrequently compared to +the main 'nifi' code tree. + +- Build with `mvn clean install` +- Presuming you need to make use of changes to the nifi-nar-maven-plugin module, you should next + go to the [nifi](../nifi) directory and follow its instructions. + ## Getting Help If you have questions, you can reach out to our mailing list: [email protected] @@ -36,9 +45,6 @@ We're also often available in IRC: #nifi on [irc.freenode.net](http://webchat.freenode.net/?channels=#nifi). -## Requirements -* JDK 1.7 or higher - ## License Except as otherwise noted this software is licensed under the http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi-parent/README.md ---------------------------------------------------------------------- diff --git a/nifi-parent/README.md b/nifi-parent/README.md index a7dd1ed..d2c10d8 100644 --- a/nifi-parent/README.md +++ b/nifi-parent/README.md @@ -18,16 +18,24 @@ Apache NiFi Parent provides a nice aggregation of shared concerns for all NiFi P ## Table of Contents +- [Requirements](#requirements) - [Getting Started](#getting-started) - [License](#license) - [Disclaimer](#disclaimer) +## Requirements +* JDK 1.7 or higher +* Apache Maven 3.0.5 or higher + ## Getting Started -- Build the nifi-parent module. Then you can go to the 'nifi-nar-maven-plugin' -directory and follow its instructions and finally the 'nifi' directory. Building -the nifi-parent and nifi-nar-maven-plugin should be rarely necessary as these -will be released infrequently by comparison to the 'nifi' tree. +Building the nifi-parent module should be rare since it will be released infrequently compared to +the main 'nifi' code tree. + +- Build the nifi-parent module using `mvn clean install`. +- Presuming you need to make use of changes to the nifi-parent module, you should next + go to the [nifi-nar-maven-plugin](../nifi-nar-maven-plugin) directory and follow its instructions + and finally the [nifi](../nifi) directory. ## Documentation http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/README.md ---------------------------------------------------------------------- diff --git a/nifi/README.md b/nifi/README.md index 5545539..cffa09a 100644 --- a/nifi/README.md +++ b/nifi/README.md @@ -19,9 +19,9 @@ Apache NiFi is an easy to use, powerful, and reliable system to process and dist ## Table of Contents - [Features](#features) +- [Requirements](#requirements) - [Getting Started](#getting-started) - [Getting Help](#getting-help) -- [Requirements](#requirements) - [License](#license) - [Disclaimer](#disclaimer) - [Export Control] (#export-control) @@ -47,17 +47,64 @@ Apache NiFi was made for dataflow. It supports highly configurable directed grap - SSL, SSH, HTTPS, encrypted content, etc... - Pluggable role-based authentication/authorization +## Requirements +* JDK 1.7 or higher +* Apache Maven 3.0.5 or higher + ## Getting Started To build: -- Execute 'mvn clean install' or for parallel build execute 'mvn -T 2.0C clean install' +- Execute `mvn clean install` or for parallel build execute `mvn -T 2.0C clean install`. On a + modest development laptop that is a couple of years old, the latter build takes a bit under ten + minutes. After a large amount of output you should eventually see a success message. + + laptop:nifi fhampton$ mvn -T 2.0C clean install + [INFO] Scanning for projects... + [INFO] Inspecting build with total of 115 modules... + ...tens of thousands of lines elided... + [INFO] ------------------------------------------------------------------------ + [INFO] BUILD SUCCESS + [INFO] ------------------------------------------------------------------------ + [INFO] Total time: 09:24 min (Wall Clock) + [INFO] Finished at: 2015-04-30T00:30:36-05:00 + [INFO] Final Memory: 173M/1359M + [INFO] ------------------------------------------------------------------------ + +To deploy: - Change directory to 'nifi-assembly'. In the target directory, there should be a build of nifi. -- Unpack the build wherever you like or use the already unpacked build. -To start NiFi: -- Change directory to the location where you installed NiFi -- '<install_location>/bin/nifi.sh start' -- Direct your browser to http://localhost:8080/nifi/ + laptop:nifi fhampton$ cd nifi-assembly + laptop:nifi-assembly fhampton$ ls -lhd target/nifi* + drwxr-xr-x 3 fhampton staff 102B Apr 30 00:29 target/nifi-0.1.0-incubating-SNAPSHOT-bin + -rw-r--r-- 1 fhampton staff 144M Apr 30 00:30 target/nifi-0.1.0-incubating-SNAPSHOT-bin.tar.gz + -rw-r--r-- 1 fhampton staff 144M Apr 30 00:30 target/nifi-0.1.0-incubating-SNAPSHOT-bin.zip + +- For testing ongoing development you could use the already unpacked build present in the directory + named "nifi-*version*-bin", where *version* is the current project version. To deploy in another + location make use of either the tarball or zipfile and unpack them wherever you like. The + distribution will be within a common parent directory named for the version. + + laptop:nifi-assembly fhampton$ mkdir ~/example-nifi-deploy + laptop:nifi-assembly fhampton$ tar xzf target/nifi-*-bin.tar.gz -C ~/example-nifi-deploy + laptop:nifi-assembly fhampton$ ls -lh ~/example-nifi-deploy/ + total 0 + drwxr-xr-x 10 fhampton staff 340B Apr 30 01:06 nifi-0.1.0-incubating-SNAPSHOT + +To run NiFi: +- Change directory to the location where you installed NiFi and run it. + + laptop:~ fhampton$ cd ~/example-nifi-deploy/nifi-* + laptop:nifi-0.1.0-incubating-SNAPSHOT fhampton$ ./bin/nifi.sh start + +- Direct your browser to http://localhost:8080/nifi/ and you should see a screen like this screenshot: +  + +- For help building your first data flow see the [NiFi User Guide](http://nifi.incubator.apache.org/docs/nifi-docs/user-guide.html) + +- If you are testing ongoing development, you will likely want to stop your instance. + + laptop:~ fhampton$ cd ~/example-nifi-deploy/nifi-* + laptop:nifi-0.1.0-incubating-SNAPSHOT fhampton$ ./bin/nifi.sh stop ## Getting Help If you have questions, you can reach out to our mailing list: [email protected] @@ -65,9 +112,6 @@ If you have questions, you can reach out to our mailing list: [email protected] We're also often available in IRC: #nifi on [irc.freenode.net](http://webchat.freenode.net/?channels=#nifi). -## Requirements -* JDK 1.7 or higher - ## License Except as otherwise noted this software is licensed under the http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-api/src/main/java/org/apache/nifi/annotation/notification/OnPrimaryNodeStateChange.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-api/src/main/java/org/apache/nifi/annotation/notification/OnPrimaryNodeStateChange.java b/nifi/nifi-api/src/main/java/org/apache/nifi/annotation/notification/OnPrimaryNodeStateChange.java new file mode 100644 index 0000000..4ea2170 --- /dev/null +++ b/nifi/nifi-api/src/main/java/org/apache/nifi/annotation/notification/OnPrimaryNodeStateChange.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.annotation.notification; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * <p> + * Marker annotation that a component can use to indicate that a method should be + * called whenever the state of the Primary Node in a cluster has changed. + * </p> + * + * <p> + * Methods with this annotation should take either no arguments or one argument of type + * {@link PrimaryNodeState}. The {@link PrimaryNodeState} provides context about what changed + * so that the component can take appropriate action. + * </p> + */ +@Documented +@Target({ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +public @interface OnPrimaryNodeStateChange { + +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-api/src/main/java/org/apache/nifi/annotation/notification/PrimaryNodeState.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-api/src/main/java/org/apache/nifi/annotation/notification/PrimaryNodeState.java b/nifi/nifi-api/src/main/java/org/apache/nifi/annotation/notification/PrimaryNodeState.java new file mode 100644 index 0000000..0d65b65 --- /dev/null +++ b/nifi/nifi-api/src/main/java/org/apache/nifi/annotation/notification/PrimaryNodeState.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.annotation.notification; + +/** + * Represents a state change that occurred for the Primary Node of a NiFi cluster. + */ +public enum PrimaryNodeState { + /** + * The node receiving this state has been elected the Primary Node of the NiFi cluster. + */ + ELECTED_PRIMARY_NODE, + + /** + * The node receiving this state was the Primary Node but has now had its Primary Node + * role revoked. + */ + PRIMARY_NODE_REVOKED; +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-assembly/NOTICE ---------------------------------------------------------------------- diff --git a/nifi/nifi-assembly/NOTICE b/nifi/nifi-assembly/NOTICE index 790e6ce..0edfd96 100644 --- a/nifi/nifi-assembly/NOTICE +++ b/nifi/nifi-assembly/NOTICE @@ -69,6 +69,9 @@ The following binary components are provided under the Apache Software License v Apache HttpCore Copyright 2005-2014 The Apache Software Foundation + Apache HttpMime + Copyright 1999-2013 The Apache Software Foundation + This project contains annotations derived from JCIP-ANNOTATIONS Copyright (c) 2005 Brian Goetz and Tim Peierls. See http://www.jcip.net @@ -518,7 +521,15 @@ The following binary components are provided under the Apache Software License v The following NOTICE information applies: GeoIP2 Java API This software is Copyright (c) 2013 by MaxMind, Inc. - + + (ASLv2) Woodstox Core ASL + The following NOTICE information applies: + This product currently only contains code developed by authors + of specific components, as identified by the source code files. + + Since product implements StAX API, it has dependencies to StAX API + classes. + (ASLv2) Amazon Web Services SDK The following NOTICE information applies: Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/nifi/nifi-assembly/pom.xml b/nifi/nifi-assembly/pom.xml index 7b08ed4..f25f0a1 100644 --- a/nifi/nifi-assembly/pom.xml +++ b/nifi/nifi-assembly/pom.xml @@ -1,13 +1,13 @@ <?xml version="1.0" encoding="UTF-8"?> -<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor -license agreements. See the NOTICE file distributed with this work for additional -information regarding copyright ownership. The ASF licenses this file to -You under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. You may obtain a copy of -the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required -by applicable law or agreed to in writing, software distributed under the -License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS -OF ANY KIND, either express or implied. See the License for the specific +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor +license agreements. See the NOTICE file distributed with this work for additional +information regarding copyright ownership. The ASF licenses this file to +You under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +by applicable law or agreed to in writing, software distributed under the +License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> @@ -165,6 +165,11 @@ language governing permissions and limitations under the License. --> </dependency> <dependency> <groupId>org.apache.nifi</groupId> + <artifactId>nifi-solr-nar</artifactId> + <type>nar</type> + </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> <artifactId>nifi-social-media-nar</artifactId> <version>0.1.0-incubating-SNAPSHOT</version> <type>nar</type> @@ -297,7 +302,7 @@ language governing permissions and limitations under the License. --> <nifi.security.ocsp.responder.url /> <nifi.security.ocsp.responder.certificate /> - <!-- nifi.properties: cluster common properties (cluster manager and nodes + <!-- nifi.properties: cluster common properties (cluster manager and nodes must have same values) --> <nifi.cluster.protocol.heartbeat.interval>5 sec</nifi.cluster.protocol.heartbeat.interval> <nifi.cluster.protocol.is.secure>false</nifi.cluster.protocol.is.secure> @@ -310,7 +315,7 @@ language governing permissions and limitations under the License. --> <nifi.cluster.protocol.multicast.service.locator.attempts>3</nifi.cluster.protocol.multicast.service.locator.attempts> <nifi.cluster.protocol.multicast.service.locator.attempts.delay>1 sec</nifi.cluster.protocol.multicast.service.locator.attempts.delay> - <!-- nifi.properties: cluster node properties (only configure for cluster + <!-- nifi.properties: cluster node properties (only configure for cluster nodes) --> <nifi.cluster.is.node>false</nifi.cluster.is.node> <nifi.cluster.node.address /> @@ -319,7 +324,7 @@ language governing permissions and limitations under the License. --> <nifi.cluster.node.unicast.manager.address /> <nifi.cluster.node.unicast.manager.protocol.port /> - <!-- nifi.properties: cluster manager properties (only configure for cluster + <!-- nifi.properties: cluster manager properties (only configure for cluster manager) --> <nifi.cluster.is.manager>false</nifi.cluster.is.manager> <nifi.cluster.manager.address /> http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java index 4e3a502..43f02ca 100644 --- a/nifi/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +++ b/nifi/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java @@ -16,9 +16,6 @@ */ package org.apache.nifi.util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; @@ -27,10 +24,15 @@ import java.net.InetSocketAddress; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class NiFiProperties extends Properties { private static final long serialVersionUID = 2119177359005492702L; @@ -50,6 +52,7 @@ public class NiFiProperties extends Properties { public static final String AUTO_RESUME_STATE = "nifi.flowcontroller.autoResumeState"; public static final String FLOW_CONTROLLER_GRACEFUL_SHUTDOWN_PERIOD = "nifi.flowcontroller.graceful.shutdown.period"; public static final String NAR_LIBRARY_DIRECTORY = "nifi.nar.library.directory"; + public static final String NAR_LIBRARY_DIRECTORY_PREFIX = "nifi.nar.library.directory."; public static final String NAR_WORKING_DIRECTORY = "nifi.nar.working.directory"; public static final String COMPONENT_DOCS_DIRECTORY = "nifi.documentation.working.directory"; public static final String SENSITIVE_PROPS_KEY = "nifi.sensitive.props.key"; @@ -230,21 +233,27 @@ public class NiFiProperties extends Properties { * obtained. * * @return the NiFiProperties object to use - * @throws RuntimeException if unable to load properties file + * @throws RuntimeException + * if unable to load properties file */ public static synchronized NiFiProperties getInstance() { if (null == instance) { final NiFiProperties suspectInstance = new NiFiProperties(); - final String nfPropertiesFilePath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH); + final String nfPropertiesFilePath = System + .getProperty(NiFiProperties.PROPERTIES_FILE_PATH); if (null == nfPropertiesFilePath || nfPropertiesFilePath.trim().length() == 0) { - throw new RuntimeException("Requires a system property called \'" + NiFiProperties.PROPERTIES_FILE_PATH + "\' and this is not set or has no value"); + throw new RuntimeException("Requires a system property called \'" + + NiFiProperties.PROPERTIES_FILE_PATH + + "\' and this is not set or has no value"); } final File propertiesFile = new File(nfPropertiesFilePath); if (!propertiesFile.exists()) { - throw new RuntimeException("Properties file doesn't exist \'" + propertiesFile.getAbsolutePath() + "\'"); + throw new RuntimeException("Properties file doesn't exist \'" + + propertiesFile.getAbsolutePath() + "\'"); } if (!propertiesFile.canRead()) { - throw new RuntimeException("Properties file exists but cannot be read \'" + propertiesFile.getAbsolutePath() + "\'"); + throw new RuntimeException("Properties file exists but cannot be read \'" + + propertiesFile.getAbsolutePath() + "\'"); } InputStream inStream = null; try { @@ -252,7 +261,8 @@ public class NiFiProperties extends Properties { suspectInstance.load(inStream); } catch (final Exception ex) { LOG.error("Cannot load properties file due to " + ex.getLocalizedMessage()); - throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex); + throw new RuntimeException("Cannot load properties file due to " + + ex.getLocalizedMessage(), ex); } finally { if (null != inStream) { try { @@ -374,7 +384,7 @@ public class NiFiProperties extends Properties { if ("false".equalsIgnoreCase(secureVal)) { return false; - }else{ + } else { return true; } @@ -406,7 +416,8 @@ public class NiFiProperties extends Properties { * @return Whether to auto start the processors or not */ public boolean getAutoResumeState() { - final String rawAutoResumeState = getProperty(AUTO_RESUME_STATE, DEFAULT_AUTO_RESUME_STATE.toString()); + final String rawAutoResumeState = getProperty(AUTO_RESUME_STATE, + DEFAULT_AUTO_RESUME_STATE.toString()); return Boolean.parseBoolean(rawAutoResumeState); } @@ -417,7 +428,8 @@ public class NiFiProperties extends Properties { * @return the number of partitions */ public int getFlowFileRepositoryPartitions() { - final String rawProperty = getProperty(FLOWFILE_REPOSITORY_PARTITIONS, DEFAULT_FLOWFILE_REPO_PARTITIONS); + final String rawProperty = getProperty(FLOWFILE_REPOSITORY_PARTITIONS, + DEFAULT_FLOWFILE_REPO_PARTITIONS); return Integer.parseInt(rawProperty); } @@ -428,7 +440,8 @@ public class NiFiProperties extends Properties { * @return the number of milliseconds between checkpoint events */ public String getFlowFileRepositoryCheckpointInterval() { - return getProperty(FLOWFILE_REPOSITORY_CHECKPOINT_INTERVAL, DEFAULT_FLOWFILE_CHECKPOINT_INTERVAL); + return getProperty(FLOWFILE_REPOSITORY_CHECKPOINT_INTERVAL, + DEFAULT_FLOWFILE_CHECKPOINT_INTERVAL); } /** @@ -470,7 +483,8 @@ public class NiFiProperties extends Properties { } public String getUserCredentialCacheDuration() { - return getProperty(SECURITY_USER_CREDENTIAL_CACHE_DURATION, DEFAULT_USER_CREDENTIAL_CACHE_DURATION); + return getProperty(SECURITY_USER_CREDENTIAL_CACHE_DURATION, + DEFAULT_USER_CREDENTIAL_CACHE_DURATION); } public boolean getSupportNewAccountRequests() { @@ -525,8 +539,28 @@ public class NiFiProperties extends Properties { return new File(getNarWorkingDirectory(), "extensions"); } - public File getNarLibraryDirectory() { - return new File(getProperty(NAR_LIBRARY_DIRECTORY, DEFAULT_NAR_LIBRARY_DIR)); + public List<Path> getNarLibraryDirectories() { + + List<Path> narLibraryPaths = new ArrayList<>(); + + // go through each property + for (String propertyName : stringPropertyNames()) { + // determine if the property is a nar library path + if (StringUtils.startsWith(propertyName, NAR_LIBRARY_DIRECTORY_PREFIX) + || NAR_LIBRARY_DIRECTORY.equals(propertyName)) { + // attempt to resolve the path specified + String narLib = getProperty(propertyName); + if (!StringUtils.isBlank(narLib)) { + narLibraryPaths.add(Paths.get(narLib)); + } + } + } + + if (narLibraryPaths.isEmpty()) { + narLibraryPaths.add(Paths.get(DEFAULT_NAR_LIBRARY_DIR)); + } + + return narLibraryPaths; } // getters for ui properties // @@ -559,7 +593,8 @@ public class NiFiProperties extends Properties { // getters for cluster protocol properties // public String getClusterProtocolHeartbeatInterval() { - return getProperty(CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL, DEFAULT_CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL); + return getProperty(CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL, + DEFAULT_CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL); } public String getNodeHeartbeatInterval() { @@ -571,7 +606,8 @@ public class NiFiProperties extends Properties { } public String getClusterProtocolConnectionHandshakeTimeout() { - return getProperty(CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT, DEFAULT_CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT); + return getProperty(CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT, + DEFAULT_CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT); } public boolean getClusterProtocolUseMulticast() { @@ -593,7 +629,8 @@ public class NiFiProperties extends Properties { } public File getPersistentStateDirectory() { - final String dirName = getProperty(PERSISTENT_STATE_DIRECTORY, DEFAULT_PERSISTENT_STATE_DIRECTORY); + final String dirName = getProperty(PERSISTENT_STATE_DIRECTORY, + DEFAULT_PERSISTENT_STATE_DIRECTORY); final File file = new File(dirName); if (!file.exists()) { file.mkdirs(); @@ -603,14 +640,16 @@ public class NiFiProperties extends Properties { public int getClusterProtocolMulticastServiceLocatorAttempts() { try { - return Integer.parseInt(getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS)); + return Integer + .parseInt(getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS)); } catch (NumberFormatException nfe) { return DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS; } } public String getClusterProtocolMulticastServiceLocatorAttemptsDelay() { - return getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY, DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY); + return getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY, + DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY); } // getters for cluster node properties // @@ -653,7 +692,8 @@ public class NiFiProperties extends Properties { if (StringUtils.isBlank(socketAddress)) { socketAddress = "localhost"; } - int socketPort = Integer.parseInt(getProperty(CLUSTER_NODE_UNICAST_MANAGER_PROTOCOL_PORT)); + int socketPort = Integer + .parseInt(getProperty(CLUSTER_NODE_UNICAST_MANAGER_PROTOCOL_PORT)); return InetSocketAddress.createUnresolved(socketAddress, socketPort); } catch (Exception ex) { throw new RuntimeException("Invalid unicast manager address/port due to: " + ex, ex); @@ -704,11 +744,13 @@ public class NiFiProperties extends Properties { } public String getClusterManagerNodeApiConnectionTimeout() { - return getProperty(CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT, DEFAULT_CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT); + return getProperty(CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT, + DEFAULT_CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT); } public String getClusterManagerNodeApiReadTimeout() { - return getProperty(CLUSTER_MANAGER_NODE_API_READ_TIMEOUT, DEFAULT_CLUSTER_MANAGER_NODE_API_READ_TIMEOUT); + return getProperty(CLUSTER_MANAGER_NODE_API_READ_TIMEOUT, + DEFAULT_CLUSTER_MANAGER_NODE_API_READ_TIMEOUT); } public int getClusterManagerNodeApiRequestThreads() { @@ -720,7 +762,8 @@ public class NiFiProperties extends Properties { } public String getClusterManagerFlowRetrievalDelay() { - return getProperty(CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY, DEFAULT_CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY); + return getProperty(CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY, + DEFAULT_CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY); } public int getClusterManagerProtocolThreads() { @@ -732,7 +775,8 @@ public class NiFiProperties extends Properties { } public String getClusterManagerSafeModeDuration() { - return getProperty(CLUSTER_MANAGER_SAFEMODE_DURATION, DEFAULT_CLUSTER_MANAGER_SAFEMODE_DURATION); + return getProperty(CLUSTER_MANAGER_SAFEMODE_DURATION, + DEFAULT_CLUSTER_MANAGER_SAFEMODE_DURATION); } public String getClusterProtocolManagerToNodeApiScheme() { @@ -780,7 +824,8 @@ public class NiFiProperties extends Properties { * configured. No directories will be created as a result of this operation. * * @return database repository path - * @throws InvalidPathException If the configured path is invalid + * @throws InvalidPathException + * If the configured path is invalid */ public Path getDatabaseRepositoryPath() { return Paths.get(getProperty(REPOSITORY_DATABASE_DIRECTORY)); @@ -791,7 +836,8 @@ public class NiFiProperties extends Properties { * configured. No directories will be created as a result of this operation. * * @return database repository path - * @throws InvalidPathException If the configured path is invalid + * @throws InvalidPathException + * If the configured path is invalid */ public Path getFlowFileRepositoryPath() { return Paths.get(getProperty(FLOWFILE_REPOSITORY_DIRECTORY)); @@ -804,7 +850,8 @@ public class NiFiProperties extends Properties { * operation. * * @return file repositories paths - * @throws InvalidPathException If any of the configured paths are invalid + * @throws InvalidPathException + * If any of the configured paths are invalid */ public Map<String, Path> getContentRepositoryPaths() { final Map<String, Path> contentRepositoryPaths = new HashMap<>(); @@ -814,7 +861,8 @@ public class NiFiProperties extends Properties { // determine if the property is a file repository path if (StringUtils.startsWith(propertyName, REPOSITORY_CONTENT_PREFIX)) { // get the repository key - final String key = StringUtils.substringAfter(propertyName, REPOSITORY_CONTENT_PREFIX); + final String key = StringUtils.substringAfter(propertyName, + REPOSITORY_CONTENT_PREFIX); // attempt to resolve the path specified contentRepositoryPaths.put(key, Paths.get(getProperty(propertyName))); @@ -839,7 +887,8 @@ public class NiFiProperties extends Properties { // determine if the property is a file repository path if (StringUtils.startsWith(propertyName, PROVENANCE_REPO_DIRECTORY_PREFIX)) { // get the repository key - final String key = StringUtils.substringAfter(propertyName, PROVENANCE_REPO_DIRECTORY_PREFIX); + final String key = StringUtils.substringAfter(propertyName, + PROVENANCE_REPO_DIRECTORY_PREFIX); // attempt to resolve the path specified provenanceRepositoryPaths.put(key, Paths.get(getProperty(propertyName))); http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java b/nifi/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java new file mode 100644 index 0000000..58fbbf3 --- /dev/null +++ b/nifi/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java @@ -0,0 +1,95 @@ +package org.apache.nifi.util; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.List; + +import org.junit.Test; + +public class NiFiPropertiesTest { + + @Test + public void testProperties() { + + NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.properties"); + + assertEquals("UI Banner Text", properties.getBannerText()); + + List<Path> directories = properties.getNarLibraryDirectories(); + + assertEquals(new File("./target/resources/NiFiProperties/lib/").getPath(), + directories.get(0).toString()); + assertEquals(new File("./target/resources/NiFiProperties/lib2/").getPath(), directories + .get(1).toString()); + + } + + @Test + public void testMissingProperties() { + + NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.missing.properties"); + + List<Path> directories = properties.getNarLibraryDirectories(); + + assertEquals(1, directories.size()); + + assertEquals(new File(NiFiProperties.DEFAULT_NAR_LIBRARY_DIR).getPath(), directories.get(0) + .toString()); + + } + + @Test + public void testBlankProperties() { + + NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.blank.properties"); + + List<Path> directories = properties.getNarLibraryDirectories(); + + assertEquals(1, directories.size()); + + assertEquals(new File(NiFiProperties.DEFAULT_NAR_LIBRARY_DIR).getPath(), directories.get(0) + .toString()); + + } + + private NiFiProperties loadSpecifiedProperties(String propertiesFile) { + + String file = NiFiPropertiesTest.class.getResource(propertiesFile).getFile(); + + System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, file); + + NiFiProperties properties = NiFiProperties.getInstance(); + + // clear out existing properties + for (String prop : properties.stringPropertyNames()) { + properties.remove(prop); + } + + InputStream inStream = null; + try { + inStream = new BufferedInputStream(new FileInputStream(file)); + properties.load(inStream); + } catch (final Exception ex) { + throw new RuntimeException("Cannot load properties file due to " + + ex.getLocalizedMessage(), ex); + } finally { + if (null != inStream) { + try { + inStream.close(); + } catch (final Exception ex) { + /** + * do nothing * + */ + } + } + } + + return properties; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties b/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties new file mode 100644 index 0000000..1f853b9 --- /dev/null +++ b/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.blank.properties @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory= +nifi.custom.nar.library.directory.alt= +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=5 mins +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=key +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.authorizedUsers.file=./target/conf/authorized-users.xml +nifi.security.user.credential.cache.duration=24 hours +nifi.security.user.authority.provider=nifi.authorization.FileAuthorizationProvider +nifi.security.support.new.account.requests= +nifi.security.default.user.roles= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties b/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties new file mode 100644 index 0000000..dbb8978 --- /dev/null +++ b/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.missing.properties @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=5 mins +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=key +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.authorizedUsers.file=./target/conf/authorized-users.xml +nifi.security.user.credential.cache.duration=24 hours +nifi.security.user.authority.provider=nifi.authorization.FileAuthorizationProvider +nifi.security.support.new.account.requests= +nifi.security.default.user.roles= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties b/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties new file mode 100644 index 0000000..ed44b88 --- /dev/null +++ b/nifi/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/nifi.properties @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=5 mins +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=key +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.authorizedUsers.file=./target/conf/authorized-users.xml +nifi.security.user.credential.cache.duration=24 hours +nifi.security.user.authority.provider=nifi.authorization.FileAuthorizationProvider +nifi.security.support.new.account.requests= +nifi.security.default.user.roles= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java b/nifi/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java index 741fdde..2c7cf23 100644 --- a/nifi/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java +++ b/nifi/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/EncryptionMethod.java @@ -46,7 +46,10 @@ public enum EncryptionMethod { SHA_256AES("PBEWITHSHAAND256BITAES-CBC-BC", "BC", true), SHA_3KEYTRIPLEDES("PBEWITHSHAAND3-KEYTRIPLEDES-CBC", "BC", true), SHA_TWOFISH("PBEWITHSHAANDTWOFISH-CBC", "BC", true), - SHA_128RC4("PBEWITHSHAAND128BITRC4", "BC", true); + SHA_128RC4("PBEWITHSHAAND128BITRC4", "BC", true), + PGP("PGP", "BC", false), + PGP_ASCII_ARMOR("PGP-ASCII-ARMOR", "BC", false); + private final String algorithm; private final String provider; private final boolean unlimitedStrength; http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnectionPool.java ---------------------------------------------------------------------- diff --git a/nifi/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnectionPool.java b/nifi/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnectionPool.java index 36d8bac..b162b3f 100644 --- a/nifi/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnectionPool.java +++ b/nifi/nifi-commons/nifi-site-to-site-client/src/main/java/org/apache/nifi/remote/client/socket/EndpointConnectionPool.java @@ -258,7 +258,7 @@ public class EndpointConnectionPool { } final PeerDescription peerDescription = peerStatus.getPeerDescription(); - BlockingQueue<EndpointConnection> connectionQueue = connectionQueueMap.get(peerStatus); + BlockingQueue<EndpointConnection> connectionQueue = connectionQueueMap.get(peerDescription); if (connectionQueue == null) { connectionQueue = new LinkedBlockingQueue<>(); BlockingQueue<EndpointConnection> existing = connectionQueueMap.putIfAbsent(peerDescription, connectionQueue); http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-docs/src/main/asciidoc/administration-guide.adoc ---------------------------------------------------------------------- diff --git a/nifi/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi/nifi-docs/src/main/asciidoc/administration-guide.adoc index d33b4de..039e5b9 100644 --- a/nifi/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -128,26 +128,74 @@ For the partitions handling the various NiFi repos turn off things like 'atime'. Doing so can cause a surprising bump in throughput. Edit the '/etc/fstab' file and for the partition(s) of interest add the 'noatime' option. + Security Configuration ---------------------- -NOTE: This section is incomplete. + +NiFi provides several different configuration options for security purposes. The most important properties are those under the +"security properties" heading in the _nifi.properties_ file. In order to run securely, the following properties must be set: + +[options="header,footer"] +|================================================================================================================================================== +| Property Name | Description +|`nifi.security.keystore` | Filename of the Keystore that contains the server's private key. +|`nifi.security.keystoreType` | The type of Keystore. Must be either `PKCS12` or `JKS`. +|`nifi.security.keystorePasswd` | The password for the Keystore. +|`nifi.security.keyPasswd` | The password for the certificate in the Keystore. If not set, the value of `nifi.security.keystorePasswd` will be used. +|`nifi.security.truststore` | Filename of the Truststore that will be used to authorize those connecting to NiFi. If not set, all who + attempt to connect will be provided access as the 'Anonymous' user. +|`nifi.security.truststoreType` | The type of the Truststore. Must be either `PKCS12` or `JKS`. +|`nifi.security.truststorePasswd` | The password for the Truststore. +|`nifi.security.needClientAuth` | Specifies whether or not connecting clients must authenticate themselves. If the Truststore properties are not set, + this must be `false`. Otherwise, a value of `true` indicates that users will be authenticated and must have + certificates that are trusted by the Truststore loaded into their web browsers. A value of `false` indicates + that all users should be given access as the 'Anonymous' user. +|================================================================================================================================================== + +Once the above properties have been configured, we can enable the User Interface to be accessed over HTTPS instead of HTTP. This is accomplished +by setting the `nifi.web.https.host` and `nifi.web.https.port` properties. The `nifi.web.https.host` property indicates which hostname the server +should run on. This allows admins to configure the application to run only on specific network interfaces. If it is desired that the HTTPS interface +be accessible from all network interfaces, a value of `0.0.0.0` should be used. + +NOTE: It is important when enabling HTTPS that the `nifi.web.http.port` property be unset. + +Now that the User Interface has been secured, we can easily secure Site-to-Site connections and inner-cluster communications, as well. This is +accomplished by setting the `nifi.remote.input.secure` and `nifi.cluster.protocol.is.secure` properties, respectively, to `true`. Controlling Levels of Access ---------------------------- Once NiFi is configured to run securely as discussed in the previous section, it is necessary -to manually designate an ADMIN user in the _authorized-users.xml_ file, which is located in the -root installation's conf directory. After this ADMIN user has been added, s/he may grant access +to configure who will have access to the system and what types of access those people will have. +NiFi controls this through the user of an 'Authority Provider.' The Authority Provider is a pluggable +mechanism for providing authorizations to different users. Which Authority Provider to use is configured +using two properties in the _nifi.properties_ file. + +The `nifi.authority.provider.configuration.file` property specifies the configuration file for Authority Providers. +The `nifi.security.user.authority.provider` property indicates which of the configured Authority Providers should be +used. + +By default, the `file-provider` Authority Provider is selected and is configured to use the permissions granted in +the _authorized-users.xml_ file. This is typically sufficient for instances of NiFi that are run in "standalone" mode. +If the NiFi instance is configured to run in a cluster, the node will typically use the `cluster-node-provider` +Provider and the Cluster Manager will typically use the `cluster-ncm-provider` Provider. Both of these Providers +have a default configuration in the _authority-providers.xml_ file but are commented out. + +When using the `cluster-node-provider` Provider, all of the authorization is provided by the Cluster Manager. In this +way, the configuration only has to be maintained in one place and will be consistent across the entire cluster. + +When configuring the Cluster Manager or a standalone node, it is necessary to manually designate an ADMIN user +in the _authorized-users.xml_ file, which is located in the root installation's conf directory. +After this ADMIN user has been added, s/he may grant access to other users, systems, and other instances of NiFi, through the User Interface (UI) without having to manually edit the _authorized-users.xml_ file. If you are the administrator, you would add yourself as the ADMIN user in this file. Open the _authorized-users.xml_ file in a text editor. You will notice that it includes a template to guide you, with example entries that are commented out. -It is only necessary to manually add one user, the ADMIN user, -to this file. -So, at a minimum, the following example entry should be included and contain the user Distinguished Name (dn) +It is only necessary to manually add one user, the ADMIN user, to this file. +So, at a minimum, the following example entry should be included and contain the user Distinguished Name (DN) in place of "user dn - read only and admin": ---- @@ -170,7 +218,7 @@ Here is an example entry using the name John Smith: After the _authorized-users.xml_ file has been edited and saved, restart NiFi. Once the application starts, the ADMIN user is -able to access the UI at the https URL that is configured in the _nifi.properties_ file. +able to access the UI at the HTTPS URL that is configured in the _nifi.properties_ file. From the UI, click on the Users icon ( image:iconUsers.png["Users", width=32] ) in the Management Toolbar (upper-right corner of the UI), and the User Management Page opens. @@ -178,7 +226,40 @@ Management Toolbar (upper-right corner of the UI), and the User Management Page The ADMIN user should be listed. Click on the pencil icon to see this user's role(s). You may edit the roles by selecting the appropriate checkboxes. -When other users want access to the NiFi UI, they navigate to the configured URL and are +The following roles are available in NiFi: + +[options="header,footer"] +|======================================================================================================== +| Role Name | Description +| Administrator | Administrator is able to configure thread pool sizes and user accounts as well as + purge the dataflow change history. +| Data Flow Manager | Data Flow Manager is given the ability to manipulate the dataflow. S/he is able to + add, remove, and manipulate components on the graph; add, remove, and manipulate + Controller Services and Reporting Tasks; create and manage templates; + view statistics; and view the bulletin board. +| Read Only | Users with Read Only access are able to view the dataflow but are unable to change anything. +| Provenance | Users with Provenance access are able to query the Data Provenance repository and view + the lineage of data. Additionally, this role provides the ability to view or download + the content of a FlowFile from a Provenance event (assuming that the content is still + available in the Content Repository and that the Authority Provider also grants access). + This access is not provided to users with Read Only + (unless the user has both Read Only and Provenance roles) because the information provided + to users with this role can potentially be very sensitive in nature, as all FlowFile attributes + and data are exposed. In order to Replay a Provenance event, a user is required to have both + the Provenance role as well as the Data Flow Manager role. +| NiFi | The NiFi Role is intended to be assigned to machines that will interact with an instance of NiFi + via Site-to-Site. This role provides the ability to send data to or retrieve data from Root + Group Ports (but only those that they are given permissions to interact with - see the User Guide + for more information on providing access to specific Ports) as well as obtain information about + which Ports exist. Note that this role allows the client to know only about the Ports that it + has permissions to interact with. +| Proxy | The Proxy Role is assigned to a system in order to grant that system permission to make requests + on behalf of a user. For instance, if an HTTP proxy service is used to gain access to the system, + the certificate being used by that service can be given the Proxy Role. +|======================================================================================================== + + +When users want access to the NiFi UI, they navigate to the configured URL and are prompted to request access. When someone has requested access, the ADMIN user sees a star on the Users icon in the Management Toolbar, alerting the ADMIN to the fact that a request is pending. Upon opening the User Management Page, the pending request is visible, and the ADMIN @@ -200,7 +281,11 @@ Clustering Configuration This section provides a quick overview of NiFi Clustering and instructions on how to set up a basic cluster. In the future, we hope to provide supplemental documentation that covers the NiFi Cluster Architecture in depth. -The design of NiFi clustering is a simple master/slave model where there is a master and one or more slaves. While the model is that of master and slave, if the master dies, the slaves are all instructed to continue operating as they were to ensure the dataflow remains live. The absence of the master simply means new slaves cannot join the cluster and cluster flow changes cannot occur until the master is restored. In NiFi clustering, we call the master the NiFi Cluster Manager (NCM), and the slaves are called Nodes. See a full description of each in the Terminology section below. +The design of NiFi clustering is a simple master/slave model where there is a master and one or more slaves. +While the model is that of master and slave, if the master dies, the slaves are all instructed to continue operating +as they were to ensure the dataflow remains live. The absence of the master simply means new slaves cannot join the +cluster and cluster flow changes cannot occur until the master is restored. In NiFi clustering, we call the master +the NiFi Cluster Manager (NCM), and the slaves are called Nodes. See a full description of each in the Terminology section below. *Why Cluster?* + @@ -217,7 +302,7 @@ NiFi Clustering is unique and has its own terminology. It's important to underst *Primary Node*: Every cluster has one Primary Node. On this node, it is possible to run "Isolated Processors" (see below). By default, the NCM will elect the first node that connects to the cluster as the Primary Node; however, the DFM may select a new node as the Primary Node in the Cluster Management page of the User Interface if desired. If the cluster restarts, the NCM will "remember" which node was the Primary Node and wait for that node to re-connect before allowing the DFM to make any changes to the dataflow. The ADMIN may adjust how long the NCM waits for the Primary Node to reconnect by adjusting the property _nifi.cluster.manager.safemode.duration_ in the _nifi.properties_ file, which is discussed in the <<system_properties>> section of this document. -*Isolated Processors*: In a NiFi cluster, the same dataflow runs on all the nodes. As a result, every component in the flow runs on every node. However, there may be cases when the DFM would not want every processor to run on every node. The most common case is when using a processor that communicates with an external service using a protocol that does not scale well. For example, the GetSFTP processor pulls from a remote directory, and if the GetSFTP on every node in the cluster tries simultaneously to pull from the same remote directory, there could be race conditions. Therefore, the DFM could configure the GetSFTP on the Primary Node to run in isolation, meaning that it only runs on that node. It could pull in data and -with the proper dataflow configuration- load-balance it across the rest of the nodes in the cluster. Note that while this feature exists, it is also very common to simply use a standalone NiFi instance to pull data and feed it to the cluster. It just depends on th e resources available and how the Administrator decides to configure the cluster. +*Isolated Processors*: In a NiFi cluster, the same dataflow runs on all the nodes. As a result, every component in the flow runs on every node. However, there may be cases when the DFM would not want every processor to run on every node. The most common case is when using a processor that communicates with an external service using a protocol that does not scale well. For example, the GetSFTP processor pulls from a remote directory, and if the GetSFTP on every node in the cluster tries simultaneously to pull from the same remote directory, there could be race conditions. Therefore, the DFM could configure the GetSFTP on the Primary Node to run in isolation, meaning that it only runs on that node. It could pull in data and -with the proper dataflow configuration- load-balance it across the rest of the nodes in the cluster. Note that while this feature exists, it is also very common to simply use a standalone NiFi instance to pull data and feed it to the cluster. It just depends on th e resources available and how the Administrator decides to configure the cluster. *Heartbeats*: The nodes communicate their health and status to the NCM via "heartbeats", which let the NCM know they are still connected to the cluster and working properly. By default, the nodes emit heartbeats to the NCM every 5 seconds, and if the NCM does not receive a heartbeat from a node within 45 seconds, it disconnects the node due to "lack of heartbeat". (The 5-second and 45-second settings are configurable in the _nifi.properties_ file. See the <<system_properties>> section of this document for more information.) The reason that the NCM disconnects the node is because the NCM needs to ensure that every node in the cluster is in sync, and if a node is not heard from regularly, the NCM cannot be sure it is still in sync with the rest of the cluster. If, after 45 seconds, the node does send a new heartbeat, the NCM will automatically reconnect the node to the cluster. Both the disconnection due to lack of heartbeat and the reconnection once a heartbeat is received are report ed to the DFM in the NCM's User Interface. http://git-wip-us.apache.org/repos/asf/incubator-nifi/blob/6f5b6225/nifi/nifi-docs/src/main/asciidoc/images/nifi_first_launch_screenshot.png ---------------------------------------------------------------------- diff --git a/nifi/nifi-docs/src/main/asciidoc/images/nifi_first_launch_screenshot.png b/nifi/nifi-docs/src/main/asciidoc/images/nifi_first_launch_screenshot.png new file mode 100644 index 0000000..7735f7e Binary files /dev/null and b/nifi/nifi-docs/src/main/asciidoc/images/nifi_first_launch_screenshot.png differ
