Merge branch 'NIFI-259'
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/4df65121 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/4df65121 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/4df65121 Branch: refs/heads/master Commit: 4df65121263fe9b3e488f81ce50d8f96e3643c21 Parents: 498b502 16f1852 Author: Aldrin Piri <[email protected]> Authored: Fri Feb 5 14:09:29 2016 -0500 Committer: Aldrin Piri <[email protected]> Committed: Fri Feb 5 14:09:29 2016 -0500 ---------------------------------------------------------------------- .../java/org/apache/nifi/action/Operation.java | 3 +- .../nifi/annotation/behavior/Stateful.java | 54 +++ .../lifecycle/OnConfigurationRestored.java | 52 ++ .../nifi/components/ConfigurableComponent.java | 14 +- .../org/apache/nifi/components/state/Scope.java | 41 ++ .../nifi/components/state/StateManager.java | 99 ++++ .../apache/nifi/components/state/StateMap.java | 51 ++ .../nifi/components/state/StateProvider.java | 133 +++++ .../StateProviderInitializationContext.java | 56 +++ .../controller/AbstractControllerService.java | 10 + .../ControllerServiceInitializationContext.java | 6 + .../AbstractSessionFactoryProcessor.java | 18 + .../apache/nifi/processor/ProcessContext.java | 6 + .../apache/nifi/reporting/ReportingContext.java | 6 + nifi-assembly/pom.xml | 6 + .../org/apache/nifi/util/NiFiProperties.java | 31 ++ .../client/socket/EndpointConnectionPool.java | 3 +- .../nifi/remote/cluster/NodeInformation.java | 16 +- .../remote/cluster/NodeInformationAdapter.java | 2 +- .../src/main/asciidoc/administration-guide.adoc | 318 ++++++++++++ .../src/main/asciidoc/developer-guide.adoc | 67 ++- .../org/apache/nifi/state/MockStateManager.java | 278 +++++++++++ .../org/apache/nifi/state/MockStateMap.java | 49 ++ ...kControllerServiceInitializationContext.java | 17 +- .../apache/nifi/util/MockProcessContext.java | 23 +- .../apache/nifi/util/MockReportingContext.java | 10 +- .../apache/nifi/util/MockValidationContext.java | 10 +- .../nifi/util/StandardProcessorTestRunner.java | 51 +- .../java/org/apache/nifi/util/TestRunner.java | 12 + .../nifi/util/TestMockProcessContext.java | 2 +- .../nifi/web/api/dto/ComponentStateDTO.java | 89 ++++ .../nifi/web/api/dto/ControllerServiceDTO.java | 15 + .../apache/nifi/web/api/dto/ProcessorDTO.java | 15 + .../nifi/web/api/dto/ReportingTaskDTO.java | 15 + .../apache/nifi/web/api/dto/StateEntryDTO.java | 90 ++++ .../apache/nifi/web/api/dto/StateMapDTO.java | 61 +++ .../web/api/entity/ComponentStateEntity.java | 44 ++ ...kControllerServiceInitializationContext.java | 8 +- .../documentation/mock/MockProcessContext.java | 8 +- .../nifi/cluster/protocol/ClusterNodes.java | 39 ++ .../cluster/protocol/ConnectionResponse.java | 27 +- .../nifi/cluster/protocol/NodeIdentifier.java | 48 +- .../jaxb/message/AdaptedConnectionResponse.java | 10 +- .../jaxb/message/AdaptedNodeIdentifier.java | 31 +- .../jaxb/message/ConnectionResponseAdapter.java | 8 +- .../jaxb/message/NodeIdentifierAdapter.java | 6 +- .../ClusterManagerProtocolSenderImplTest.java | 6 +- .../impl/NodeProtocolSenderImplTest.java | 9 +- .../org/apache/nifi/cluster/event/Event.java | 1 + .../cluster/event/impl/EventManagerImpl.java | 1 + .../impl/FileBasedClusterNodeFirewall.java | 3 +- .../cluster/manager/HttpClusterManager.java | 14 +- .../cluster/manager/HttpRequestReplicator.java | 3 +- .../cluster/manager/HttpResponseMapper.java | 1 + .../nifi/cluster/manager/NodeResponse.java | 7 +- .../exception/ConflictingNodeIdException.java | 46 ++ .../manager/impl/ClusteredEventAccess.java | 8 +- .../manager/impl/ClusteredReportingContext.java | 12 +- .../manager/impl/HttpRequestReplicatorImpl.java | 19 +- .../manager/impl/HttpResponseMapperImpl.java | 1 + .../cluster/manager/impl/WebClusterManager.java | 245 +++++++--- .../java/org/apache/nifi/cluster/node/Node.java | 1 - ...anagerProtocolServiceLocatorFactoryBean.java | 1 - ...FileBasedClusterNodeFirewallFactoryBean.java | 1 + .../reporting/ClusteredReportingTaskNode.java | 7 +- .../event/impl/EventManagerImplTest.java | 8 +- .../impl/FileBasedClusterNodeFirewallTest.java | 8 +- .../impl/DataFlowManagementServiceImplTest.java | 27 +- .../impl/HttpRequestReplicatorImplTest.java | 38 +- .../impl/HttpResponseMapperImplTest.java | 21 +- .../manager/impl/TestWebClusterManager.java | 2 - .../cluster/manager/testutils/HttpRequest.java | 2 + .../cluster/manager/testutils/HttpResponse.java | 1 + .../cluster/manager/testutils/HttpServer.java | 1 + .../components/state/StateManagerProvider.java | 60 +++ .../apache/nifi/connectable/Connectable.java | 2 + .../controller/AbstractConfiguredComponent.java | 9 +- .../apache/nifi/controller/AbstractPort.java | 4 + .../nifi/controller/ConfiguredComponent.java | 6 + .../nifi/controller/ReportingTaskNode.java | 2 + .../apache/nifi/controller/StandardFunnel.java | 4 + .../service/ControllerServiceNode.java | 2 + .../org/apache/nifi/logging/LogRepository.java | 12 + .../nifi-framework/nifi-framework-core/pom.xml | 21 + .../apache/nifi/cluster/HeartbeatPayload.java | 18 - .../apache/nifi/controller/FlowController.java | 156 +++++- .../nifi/controller/StandardFlowService.java | 50 +- .../controller/StandardFlowSynchronizer.java | 8 +- .../nifi/controller/StandardProcessorNode.java | 5 + .../reporting/AbstractReportingTaskNode.java | 5 + .../reporting/StandardReportingContext.java | 10 +- .../reporting/StandardReportingTaskNode.java | 2 +- .../repository/BatchingSessionFactory.java | 4 +- .../scheduling/ConnectableProcessContext.java | 10 +- .../scheduling/EventDrivenSchedulingAgent.java | 21 +- .../scheduling/QuartzSchedulingAgent.java | 10 +- .../controller/scheduling/ScheduleState.java | 10 +- .../scheduling/StandardProcessScheduler.java | 31 +- .../scheduling/TimerDrivenSchedulingAgent.java | 9 +- ...dControllerServiceInitializationContext.java | 10 +- .../service/StandardControllerServiceNode.java | 5 + .../StandardControllerServiceProvider.java | 16 +- .../nifi/controller/state/ClusterState.java | 38 ++ .../controller/state/ConfigParseException.java | 30 ++ .../nifi/controller/state/NodeDescription.java | 39 ++ .../controller/state/StandardStateManager.java | 92 ++++ .../nifi/controller/state/StandardStateMap.java | 53 ++ ...ndardStateProviderInitializationContext.java | 60 +++ .../nifi/controller/state/StateMapSerDe.java | 114 +++++ .../nifi/controller/state/StateMapUpdate.java | 45 ++ .../state/StateProviderException.java | 30 ++ .../state/config/StateManagerConfiguration.java | 143 ++++++ .../config/StateProviderConfiguration.java | 53 ++ .../manager/StandardStateManagerProvider.java | 301 ++++++++++++ .../state/providers/AbstractStateProvider.java | 58 +++ .../local/WriteAheadLocalStateProvider.java | 265 ++++++++++ .../zookeeper/ZooKeeperStateProvider.java | 485 +++++++++++++++++++ .../state/server/ZooKeeperStateServer.java | 176 +++++++ .../nifi/groups/StandardProcessGroup.java | 20 +- .../repository/StandardLogRepository.java | 13 + .../nifi/processor/StandardProcessContext.java | 10 +- .../processor/StandardSchedulingContext.java | 10 +- ...g.apache.nifi.components.state.StateProvider | 16 + .../TestStandardProcessScheduler.java | 22 +- .../StandardControllerServiceProviderTest.java | 28 +- .../TestStandardControllerServiceProvider.java | 40 +- .../controller/state/TestStateMapSerDe.java | 67 +++ .../providers/AbstractTestStateProvider.java | 192 ++++++++ .../local/TestWriteAheadLocalStateProvider.java | 90 ++++ .../zookeeper/TestZooKeeperStateProvider.java | 205 ++++++++ .../src/main/resources/conf/nifi.properties | 14 + .../main/resources/conf/state-management.xml | 68 +++ .../main/resources/conf/zookeeper.properties | 45 ++ .../socket/ClusterManagerServerProtocol.java | 2 +- .../nifi/audit/ComponentStateAuditor.java | 185 +++++++ .../org/apache/nifi/web/NiFiServiceFacade.java | 80 +++ .../apache/nifi/web/NiFiServiceFacadeLock.java | 11 + .../nifi/web/StandardNiFiServiceFacade.java | 117 +++++ .../nifi/web/api/ControllerServiceResource.java | 234 +++++++-- .../apache/nifi/web/api/ProcessorResource.java | 146 ++++++ .../nifi/web/api/ReportingTaskResource.java | 166 +++++++ .../org/apache/nifi/web/api/dto/DtoFactory.java | 127 +++-- .../nifi/web/controller/ControllerFacade.java | 41 +- .../apache/nifi/web/dao/ComponentStateDAO.java | 74 +++ .../nifi/web/dao/ControllerServiceDAO.java | 25 + .../org/apache/nifi/web/dao/ProcessorDAO.java | 27 ++ .../apache/nifi/web/dao/ReportingTaskDAO.java | 25 + .../web/dao/impl/StandardComponentStateDAO.java | 98 ++++ .../dao/impl/StandardControllerServiceDAO.java | 27 ++ .../nifi/web/dao/impl/StandardProcessorDAO.java | 26 + .../web/dao/impl/StandardReportingTaskDAO.java | 26 + .../spring/StateManagerProviderFactoryBean.java | 74 +++ .../src/main/resources/nifi-web-api-context.xml | 18 +- .../resources/access-control/nifi.properties | 7 + .../access-control/state-management.xml | 38 ++ .../nifi-framework/nifi-web/nifi-web-ui/pom.xml | 2 + .../main/resources/filters/canvas.properties | 1 + .../src/main/webapp/WEB-INF/pages/canvas.jsp | 1 + .../partials/canvas/component-state-dialog.jsp | 45 ++ .../nifi-web-ui/src/main/webapp/css/canvas.css | 1 + .../src/main/webapp/css/component-state.css | 92 ++++ .../css/new-controller-service-dialog.css | 6 +- .../main/webapp/css/new-processor-dialog.css | 6 +- .../webapp/css/new-reporting-task-dialog.css | 6 +- .../src/main/webapp/images/iconViewState.png | Bin 0 -> 1245 bytes .../src/main/webapp/js/nf/canvas/nf-actions.js | 17 + .../src/main/webapp/js/nf/canvas/nf-canvas.js | 1 + .../webapp/js/nf/canvas/nf-component-state.js | 376 ++++++++++++++ .../main/webapp/js/nf/canvas/nf-context-menu.js | 25 + .../js/nf/canvas/nf-processor-configuration.js | 21 +- .../src/main/webapp/js/nf/canvas/nf-settings.js | 171 ++++--- .../apache/nifi/processors/hadoop/ListHDFS.java | 243 ++++------ .../processors/hadoop/util/HDFSListing.java | 45 ++ .../nifi/processors/hadoop/TestListHDFS.java | 58 ++- .../java/org/apache/nifi/hbase/GetHBase.java | 233 ++++++--- .../org/apache/nifi/hbase/TestGetHBase.java | 93 ++-- .../processors/hl7/ExtractHL7Attributes.java | 3 +- .../apache/nifi/processors/hl7/RouteHL7.java | 2 +- .../processors/kafka/test/EmbeddedKafka.java | 3 +- .../processors/script/TestInvokeGroovy.java | 6 +- .../standard/AbstractListProcessor.java | 328 +++++++------ .../nifi/processors/standard/GetHTTP.java | 202 ++++---- .../nifi/processors/standard/ListFile.java | 29 ++ .../nifi/processors/standard/ListSFTP.java | 13 + .../nifi/processors/standard/TailFile.java | 259 ++++++---- .../standard/TestAbstractListProcessor.java | 78 ++- .../standard/TestDetectDuplicate.java | 42 +- .../nifi/processors/standard/TestGetHTTP.java | 161 +++--- .../nifi/processors/standard/TestListFile.java | 14 - .../standard/TestRouteOnAttribute.java | 5 +- pom.xml | 118 +++-- 191 files changed, 8744 insertions(+), 1312 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-assembly/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-docs/src/main/asciidoc/administration-guide.adoc ---------------------------------------------------------------------- diff --cc nifi-docs/src/main/asciidoc/administration-guide.adoc index 5987934,a6da382..5151667 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@@ -366,200 -366,8 +366,201 @@@ a Remote Process Group. In that scenari in the remote cluster can be included in the same group. When the ADMIN wants to grant port access to the remote cluster, s/he can grant it to the group and avoid having to grant it individually to each node in the cluster. +[[encryption]] +Encryption Configuration +------------------------ + +This section provides an overview of the capabilities of NiFi to encrypt and decrypt data. + +The `EncryptContent` processor allows for the encryption and decryption of data, both internal to NiFi and integrated with external systems, such as `openssl` and other data sources and consumers. + +[[key-derivation-functions]] +Key Derivation Functions +~~~~~~~~~~~~~~~~~~~~~~~~ + +Key Derivation Functions (KDF) are mechanisms by which human-readable information, usually a password or other secret information, is translated into a cryptographic key suitable for data protection. For further information, read https://en.wikipedia.org/wiki/Key_derivation_function[the Wikipedia entry on Key Derivation Functions]. +Currently, KDFs are ingested by `CipherProvider` implementations and return a fully-initialized `Cipher` object to be used for encryption or decryption. Due to the use of a `CipherProviderFactory`, the KDFs are not customizable at this time. Future enhancements will include the ability to provide custom cost parameters to the KDF at initialization time. As a work-around, `CipherProvider` instances can be initialized with custom cost parameters in the constructor but this is not currently supported by the `CipherProviderFactory`. +Here are the KDFs currently supported by NiFi (primarily in the `EncryptContent` processor for password-based encryption (PBE)) and relevant notes: + +* NiFi Legacy KDF +** The original KDF used by NiFi for internal key derivation for PBE, this is 1000 iterations of the MD5 digest over the concatenation of the password and 16 bytes of random salt. +** This KDF is *deprecated as of NiFi 0.5.0* and should only be used for backwards compatibility to decrypt data that was previously encrypted by a legacy version of NiFi. +* OpenSSL PKCS#5 v1.5 EVP_BytesToKey +** This KDF was added in v0.4.0. +** This KDF is provided for compatibility with data encrypted using OpenSSL's default PBE, known as `EVP_BytesToKey`. This is a single iteration of MD5 over the concatenation of the password and 8 bytes of random ASCII salt. OpenSSL recommends using `PBKDF2` for key derivation but does not expose the library method necessary to the command-line tool, so this KDF is still the de facto default for command-line encryption. +* Bcrypt +** This KDF was added in v0.5.0. +** https://en.wikipedia.org/wiki/Bcrypt[Bcrypt] is an adaptive function based on the https://en.wikipedia.org/wiki/Blowfish_(cipher)[Blowfish] cipher. This KDF is strongly recommended as it automatically incorporates a random 16 byte salt, configurable cost parameter (or "work factor"), and is hardened against brute-force attacks using https://en.wikipedia.org/wiki/General-purpose_computing_on_graphics_processing_units[GPGPU] (which share memory between cores) by requiring access to "large" blocks of memory during the key derivation. It is less resistant to https://en.wikipedia.org/wiki/Field-programmable_gate_array[FPGA] brute-force attacks where the gate arrays have access to individual embedded RAM blocks. +** Because the length of a Bcrypt-derived key is always 184 bits, the complete output is then fed to a `SHA-512` digest and truncated to the desired key length. This provides the benefit of the avalanche effect on the formatted input. +** The recommended minimum work factor is 12 (2^12^ key derivation rounds) (as of 2/1/2016 on commodity hardware) and should be increased to the threshold at which legitimate systems will encounter detrimental delays (see schedule below or use `BcryptCipherProviderGroovyTest#testDefaultConstructorShouldProvideStrongWorkFactor()` to calculate safe minimums). +** The salt format is `$2a$10$ABCDEFGHIJKLMNOPQRSTUV`. The salt is delimited by `$` and the three sections are as follows: +*** `2a` - the version of the format. An extensive explanation can be found http://blog.ircmaxell.com/2012/12/seven-ways-to-screw-up-bcrypt.html[here]. NiFi currently uses `2a` for all salts generated internally. +*** `10` - the work factor. This is actually the log~2~ value, so the total iteration count would be 2^10^ in this case. +*** `ABCDEFGHIJKLMNOPQRSTUV` - the 22 character, Base64-encoded, unpadded, raw salt value. This decodes to a 16 byte salt used in the key derivation. +* Scrypt +** This KDF was added in v0.5.0. +** https://en.wikipedia.org/wiki/Scrypt[Scrypt] is an adaptive function designed in response to `bcrypt`. This KDF is recommended as it requires relatively large amounts of memory for each derivation, making it resistant to hardware brute-force attacks. +** The recommended minimum cost is `N`=2^14^, `r`=8, `p`=1 (as of 2/1/2016 on commodity hardware) and should be increased to the threshold at which legitimate systems will encounter detrimental delays (see schedule below or use `ScryptCipherProviderGroovyTest#testDefaultConstructorShouldProvideStrongParameters()` to calculate safe minimums). +** The salt format is `$s0$e0101$ABCDEFGHIJKLMNOPQRSTUV`. The salt is delimited by `$` and the three sections are as follows: +*** `s0` - the version of the format. NiFi currently uses `s0` for all salts generated internally. +*** `e0101` - the cost parameters. This is actually a hexadecimal encoding of `N`, `r`, `p` using shifts. This can be formed/parsed using `Scrypt#encodeParams()` and `Scrypt#parseParameters()`. +**** Some external libraries encode `N`, `r`, and `p` separately in the form `$400$1$1$`. A utility method is available at `ScryptCipherProvider#translateSalt()` which will convert the external form to the internal form. +*** `ABCDEFGHIJKLMNOPQRSTUV` - the 11-44 character, Base64-encoded, unpadded, raw salt value. This decodes to a 8-32 byte salt used in the key derivation. +* PBKDF2 +** This KDF was added in v0.5.0. +** https://en.wikipedia.org/wiki/PBKDF2[Password-Based Key Derivation Function 2] is an adaptive derivation function which uses an internal pseudorandom function (PRF) and iterates it many times over a password and salt (at least 16 bytes). +** The PRF is recommended to be `HMAC/SHA-256` or `HMAC/SHA-512`. The use of an HMAC cryptographic hash function mitigates a length extension attack. +** The recommended minimum number of iterations is 160,000 (as of 2/1/2016 on commodity hardware). This number should be doubled every two years (see schedule below or use `PBKDF2CipherProviderGroovyTest#testDefaultConstructorShouldProvideStrongIterationCount()` to calculate safe minimums). +** This KDF is not memory-hard (can be parallelized massively with commodity hardware) but is still recommended as sufficient by http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf[NIST SP 800-132 (PDF)] and many cryptographers (when used with a proper iteration count and HMAC cryptographic hash function). +* None +** This KDF was added in v0.5.0. +** This KDF performs no operation on the input and is a marker to indicate the raw key is provided to the cipher. The key must be provided in hexadecimal encoding and be of a valid length for the associated cipher/algorithm. + +Additional Resources +^^^^^^^^^^^^^^^^^^^^ + +* http://stackoverflow.com/a/30308723/70465[Explanation of optimal scrypt cost parameters and relationships] +* http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf[NIST Special Publication 800-132] +* https://www.owasp.org/index.php/Password_Storage_Cheat_Sheet#Work_Factor[OWASP Password Storage Work Factor Calculations] +* http://security.stackexchange.com/a/3993/16485[PBKDF2 rounds calculations] +* http://blog.ircmaxell.com/2014/03/why-i-dont-recommend-scrypt.html[Scrypt as KDF vs password storage vulnerabilities] +* http://security.stackexchange.com/a/26253/16485[Scrypt vs. Bcrypt (as of 2010)] +* http://security.stackexchange.com/a/6415/16485[Bcrypt vs PBKDF2] +* http://wildlyinaccurate.com/bcrypt-choosing-a-work-factor/[Choosing a work factor for Bcrypt] +* https://docs.spring.io/spring-security/site/docs/current/apidocs/org/springframework/security/crypto/bcrypt/BCrypt.html[Spring Security Bcrypt] +* https://www.openssl.org/docs/manmaster/crypto/EVP_BytesToKey.html[OpenSSL EVP BytesToKey PKCS#1v1.5] +* https://www.openssl.org/docs/manmaster/crypto/PKCS5_PBKDF2_HMAC.html[OpenSSL PBKDF2 KDF] +* http://security.stackexchange.com/a/29139/16485[OpenSSL KDF flaws description] + +Salt and IV Encoding +~~~~~~~~~~~~~~~~~~~~ + +Initially, the `EncryptContent` processor had a single method of deriving the encryption key from a user-provided password. This is now referred to as `NiFiLegacy` mode, effectively `MD5 digest, 1000 iterations`. In v0.4.0, another method of deriving the key, `OpenSSL PKCS#5 v1.5 EVP_BytesToKey` was added for compatibility with content encrypted outside of NiFi using the `openssl` command-line tool. Both of these <<key-derivation-functions, Key Derivation Functions>> (KDF) had hard-coded digest functions and iteration counts, and the salt format was also hard-coded. With v0.5.0, additional KDFs are introduced with variable iteration counts, work factors, and salt formats. In addition, _raw keyed encryption_ was also introduced. This required the capacity to encode arbitrary salts and Initialization Vectors (IV) into the cipher stream in order to be recovered by NiFi or a follow-on system to decrypt these messages. + +For the existing KDFs, the salt format has not changed. + +NiFi Legacy +^^^^^^^^^^^ + +The first 16 bytes of the input are the salt. On decryption, the salt is read in and combined with the password to derive the encryption key and IV. + +image:nifi-legacy-salt.png["NiFi Legacy Salt Encoding"] + +OpenSSL PKCS#5 v1.5 EVP_BytesToKey +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +OpenSSL allows for salted or unsalted key derivation. _*Unsalted key derivation is a security risk and is not recommended.*_ If a salt is present, the first 8 bytes of the input are the ASCII string "`Salted__`" (`0x53 61 6C 74 65 64 5F 5F`) and the next 8 bytes are the ASCII-encoded salt. On decryption, the salt is read in and combined with the password to derive the encryption key and IV. If there is no salt header, the entire input is considered to be the cipher text. + +image:openssl-salt.png["OpenSSL Salt Encoding"] + +For new KDFs, each of which allow for non-deterministic IVs, the IV must be stored alongside the cipher text. This is not a vulnerability, as the IV is not required to be secret, but simply to be unique for messages encrypted using the same key to reduce the success of cryptographic attacks. For these KDFs, the output consists of the salt, followed by the salt delimiter, UTF-8 string "`NiFiSALT`" (`0x4E 69 46 69 53 41 4C 54`) and then the IV, followed by the IV delimiter, UTF-8 string "`NiFiIV`" (`0x4E 69 46 69 49 56`), followed by the cipher text. + +Bcrypt, Scrypt, PBKDF2 +^^^^^^^^^^^^^^^^^^^^^^ + +image:bcrypt-salt.png["Bcrypt Salt & IV Encoding"] + +image:scrypt-salt.png["Scrypt Salt & IV Encoding"] + +image:pbkdf2-salt.png["PBKDF2 Salt & IV Encoding"] + +Java Cryptography Extension (JCE) Limited Strength Jurisdiction Policies +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Because of US export regulations, default JVMs have http://docs.oracle.com/javase/7/docs/technotes/guides/security/SunProviders.html#importlimits[limits imposed on the strength of cryptographic operations] available to them. For example, AES operations are limited to `128 bit keys` by default. While `AES-128` is cryptographically safe, this can have unintended consequences, specifically on Password-based Encryption (PBE). + +PBE is the process of deriving a cryptographic key for encryption or decryption from _user-provided secret material_, usually a password. Rather than a human remembering a (random-appearing) 32 or 64 character hexadecimal string, a password or passphrase is used. + +A number of PBE algorithms provided by NiFi impose strict limits on the length of the password due to the underlying key length checks. Below is a table listing the maximum password length on a JVM with limited cryptographic strength. + +.Maximum Password Length on Limited Cryptographic Strength JVM +|=== +|Algorithm |Max Password Length + +|`PBEWITHMD5AND128BITAES-CBC-OPENSSL` +|16 + +|`PBEWITHMD5AND192BITAES-CBC-OPENSSL` +|16 + +|`PBEWITHMD5AND256BITAES-CBC-OPENSSL` +|16 + +|`PBEWITHMD5ANDDES` +|16 + +|`PBEWITHMD5ANDRC2` +|16 + +|`PBEWITHSHA1ANDRC2` +|16 + +|`PBEWITHSHA1ANDDES` +|16 + +|`PBEWITHSHAAND128BITAES-CBC-BC` +|7 + +|`PBEWITHSHAAND192BITAES-CBC-BC` +|7 + +|`PBEWITHSHAAND256BITAES-CBC-BC` +|7 + +|`PBEWITHSHAAND40BITRC2-CBC` +|7 + +|`PBEWITHSHAAND128BITRC2-CBC` +|7 + +|`PBEWITHSHAAND40BITRC4` +|7 + +|`PBEWITHSHAAND128BITRC4` +|7 + +|`PBEWITHSHA256AND128BITAES-CBC-BC` +|7 + +|`PBEWITHSHA256AND192BITAES-CBC-BC` +|7 + +|`PBEWITHSHA256AND256BITAES-CBC-BC` +|7 + +|`PBEWITHSHAAND2-KEYTRIPLEDES-CBC` +|7 + +|`PBEWITHSHAAND3-KEYTRIPLEDES-CBC` +|7 + +|`PBEWITHSHAANDTWOFISH-CBC` +|7 +|=== + +Allow Insecure Cryptographic Modes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +By default, the `Allow Insecure Cryptographic Modes` property in `EncryptContent` processor settings is set to `not-allowed`. This means that if a password of fewer than `10` characters is provided, a validation error will occur. 10 characters is a conservative estimate and does not take into consideration full entropy calculations, patterns, etc. + +image:allow-weak-crypto.png["Allow Insecure Cryptographic Modes", width=940] + +On a JVM with limited strength cryptography, some PBE algorithms limit the maximum password length to 7, and in this case it will not be possible to provide a "safe" password. It is recommended to install the JCE Unlimited Strength Jurisdiction Policy files for the JVM to mitigate this issue. + +* http://www.oracle.com/technetwork/java/javase/downloads/jce-7-download-432124.html[JCE Unlimited Strength Jurisdiction Policy files for Java 7] +* http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html[JCE Unlimited Strength Jurisdiction Policy files for Java 8] + +If on a system where the unlimited strength policies cannot be installed, it is recommended to switch to an algorithm that supports longer passwords (see table above). + +[WARNING] +.Allowing Weak Crypto +===================== +If it is not possible to install the unlimited strength jurisdiction policies, the `Allow Weak Crypto` setting can be changed to `allowed`, but *this is _not_ recommended*. Changing this setting explicitly acknowledges the inherent risk in using weak cryptographic configurations. +===================== + +It is preferable to request upstream/downstream systems to switch to https://cwiki.apache.org/confluence/display/NIFI/Encryption+Information[keyed encryption] or use a "strong" https://cwiki.apache.org/confluence/display/NIFI/Key+Derivation+Function+Explanations[Key Derivation Function (KDF) supported by NiFi]. + [[clustering]] Clustering Configuration ------------------------ http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java ---------------------------------------------------------------------- diff --cc nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java index bf5977c,4372670..7358b42 --- a/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java +++ b/nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java @@@ -576,9 -582,10 +584,11 @@@ public class StandardProcessorTestRunne // } // } - final ComponentLog logger = new MockProcessorLog(identifier, service); + final MockProcessorLog logger = new MockProcessorLog(identifier, service); + controllerServiceLoggers.put(identifier, logger); - final MockControllerServiceInitializationContext initContext = new MockControllerServiceInitializationContext(requireNonNull(service), requireNonNull(identifier), logger); + final MockStateManager serviceStateManager = new MockStateManager(service); - controllerServiceStateManagers.put(identifier, serviceStateManager); + final MockControllerServiceInitializationContext initContext = new MockControllerServiceInitializationContext(requireNonNull(service), requireNonNull(identifier), logger, serviceStateManager); ++ controllerServiceStateManagers.put(identifier, serviceStateManager); initContext.addControllerServices(context); service.initialize(initContext); @@@ -771,12 -793,19 +796,28 @@@ sharedState.clearProvenanceEvents(); } + @Override + public MockStateManager getStateManager() { + return processorStateManager; + } + + /** + * Returns the State Manager for the given Controller Service. + * + * @param controllerService the Controller Service whose State Manager should be returned + * @return the State Manager for the given Controller Service + */ + @Override + public MockStateManager getStateManager(final ControllerService controllerService) { + return controllerServiceStateManagers.get(controllerService.getIdentifier()); + } ++ + public MockProcessorLog getLogger() { + return logger; + } + + public MockProcessorLog getControllerServiceLogger(final String identifier) { + return controllerServiceLoggers.get(identifier); + } + } http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java ---------------------------------------------------------------------- diff --cc nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java index 2a4afd9,9e7082c..5e45299 --- a/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java +++ b/nifi-mock/src/main/java/org/apache/nifi/util/TestRunner.java @@@ -826,16 -827,13 +827,27 @@@ public interface TestRunner void clearProvenanceEvents(); /** + * Returns the {@link MockProcessorLog} that is used by the Processor under test. + * @return the logger + */ + public MockProcessorLog getLogger(); + + /** + * Returns the {@link MockProcessorLog} that is used by the specified controller service. + * + * @param identifier a controller service identifier + * @return the logger + */ + public MockProcessorLog getControllerServiceLogger(final String identifier); ++ ++ /** + * @return the State Manager that is used to stored and retrieve state + */ + MockStateManager getStateManager(); + + /** + * @param service the controller service of interest + * @return the State Manager that is used to store and retrieve state for the given controller service + */ + MockStateManager getStateManager(ControllerService service); } http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java ---------------------------------------------------------------------- diff --cc nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 840df94,e82c20f..dab2d3d --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@@ -176,38 -214,7 +181,39 @@@ import org.apache.zookeeper.server.quor import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.sun.jersey.api.client.ClientHandlerException; +import javax.net.ssl.SSLContext; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; ++import java.util.concurrent.locks.LockSupport; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static java.util.Objects.requireNonNull; public class FlowController implements EventAccess, ControllerServiceProvider, ReportingTaskProvider, Heartbeater, QueueProvider { @@@ -1172,9 -1241,8 +1237,8 @@@ } } } finally { - writeLock.unlock(); + readLock.unlock(); } - } /** http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/resources/filters/canvas.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas.js ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-processor-configuration.js ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js ---------------------------------------------------------------------- diff --cc nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js index c07d52c,994d78c..dcec5c0 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js @@@ -171,6 -171,12 +171,12 @@@ nf.Settings = (function () * Hides the selected controller service. */ var clearSelectedControllerService = function () { + if (nf.Canvas.isClustered()) { + $('#controller-service-availability-combo').combo('setSelectedOption', { + value: config.node + }); + } - ++ $('#controller-service-type-description').text(''); $('#controller-service-type-name').text(''); $('#selected-controller-service-name').text(''); @@@ -192,7 -198,7 +198,7 @@@ /** * Performs the filtering. -- * ++ * * @param {object} item The item subject to filtering * @param {object} args Filter arguments * @returns {Boolean} Whether or not to include the item @@@ -224,7 -230,7 +230,7 @@@ /** * Determines if the item matches the filter. -- * ++ * * @param {object} item The item to filter * @param {object} args The filter criteria * @returns {boolean} Whether the item matches the filter @@@ -250,7 -256,7 +256,7 @@@ /** * Determines if the specified tags match all the tags selected by the user. -- * ++ * * @argument {string[]} tagFilters The tag filters * @argument {string} tags The tags to test */ @@@ -293,7 -299,7 +299,7 @@@ /** * Adds a new controller service of the specified type. -- * ++ * * @param {string} controllerServiceType */ var addControllerService = function (controllerServiceType) { @@@ -478,7 -484,7 +484,7 @@@ // end the udpate controllerServiceTypesData.endUpdate(); -- ++ // set the total number of processors $('#total-controller-service-types, #displayed-controller-service-types').text(response.controllerServiceTypes.length); @@@ -544,7 -543,7 +550,7 @@@ /** * Formatter for the type column. -- * ++ * * @param {type} row * @param {type} cell * @param {type} value @@@ -558,7 -557,7 +564,7 @@@ /** * Formatter for the availability column. -- * ++ * * @param {type} row * @param {type} cell * @param {type} value @@@ -576,7 -575,7 +582,7 @@@ /** * Sorts the specified data using the specified sort details. -- * ++ * * @param {object} sortDetails * @param {object} data */ @@@ -628,24 -601,28 +634,28 @@@ // more details formatter var moreControllerServiceDetails = function (row, cell, value, columnDef, dataContext) { var markup = '<img src="images/iconDetails.png" title="View Details" class="pointer view-controller-service" style="margin-top: 5px; float: left;" />'; + + // always include a button to view the usage + markup += '<img src="images/iconUsage.png" title="Usage" class="pointer controller-service-usage" style="margin-left: 6px; margin-top: 3px; float: left;" />'; + var hasErrors = !nf.Common.isEmpty(dataContext.validationErrors); var hasBulletins = !nf.Common.isEmpty(dataContext.bulletins); -- ++ if (hasErrors) { - markup += '<img src="images/iconAlert.png" class="has-errors" style="margin-top: 4px; margin-left: 1px; float: left;" />'; + markup += '<img src="images/iconAlert.png" class="has-errors" style="margin-top: 4px; margin-left: 3px; float: left;" />'; } -- ++ if (hasBulletins) { markup += '<img src="images/iconBulletin.png" class="has-bulletins" style="margin-top: 5px; margin-left: 5px; float: left;"/>'; } -- ++ if (hasErrors || hasBulletins) { markup += '<span class="hidden row-id">' + nf.Common.escapeHtml(dataContext.id) + '</span>'; } -- ++ return markup; }; -- ++ var controllerServiceStateFormatter = function (row, cell, value, columnDef, dataContext) { // determine the appropriate label var icon = '', label = ''; @@@ -667,7 -644,7 +677,7 @@@ label = 'Enabling'; } } -- ++ // format the markup var formattedValue = '<div class="' + icon + '" style="margin-top: 3px;"></div>'; return formattedValue + '<div class="status-text" style="margin-top: 2px; margin-left: 4px; float: left;">' + label + '</div>'; @@@ -676,19 -653,23 +686,23 @@@ var controllerServiceActionFormatter = function (row, cell, value, columnDef, dataContext) { var markup = ''; - // only DFMs can edit a controller service + // only DFMs can edit a controller service and view state if (nf.Common.isDFM()) { if (dataContext.state === 'ENABLED' || dataContext.state === 'ENABLING') { - markup += '<img src="images/iconDisable.png" title="Disable" class="pointer disable-controller-service" style="margin-top: 2px;" /> '; + markup += '<img src="images/iconDisable.png" title="Disable" class="pointer disable-controller-service" style="margin-top: 2px;" />'; } else if (dataContext.state === 'DISABLED') { - markup += '<img src="images/iconEdit.png" title="Edit" class="pointer edit-controller-service" style="margin-top: 2px;" /> '; - + markup += '<img src="images/iconEdit.png" title="Edit" class="pointer edit-controller-service" style="margin-top: 2px;" />'; - ++ // if there are no validation errors allow enabling if (nf.Common.isEmpty(dataContext.validationErrors)) { - markup += '<img src="images/iconEnable.png" title="Enable" class="pointer enable-controller-service" style="margin-top: 2px;"/> '; + markup += '<img src="images/iconEnable.png" title="Enable" class="pointer enable-controller-service" style="margin-top: 2px; margin-left: 3px;"/>'; } -- - markup += '<img src="images/iconDelete.png" title="Remove" class="pointer delete-controller-service" style="margin-top: 2px;" /> '; ++ + markup += '<img src="images/iconDelete.png" title="Remove" class="pointer delete-controller-service" style="margin-top: 2px; margin-left: 3px;" />'; + } + + if (dataContext.persistsState === true) { + markup += '<img src="images/iconViewState.png" title="View State" class="pointer view-state-controller-service" style="margin-top: 2px; margin-left: 3px;" />'; } } @@@ -710,7 -689,7 +724,7 @@@ if (nf.Canvas.isClustered()) { controllerServicesColumns.push({id: 'availability', field: 'availability', name: 'Availability', formatter: availabilityFormatter, sortable: true, resizeable: true}); } -- ++ // action column should always be last controllerServicesColumns.push({id: 'actions', name: ' ', resizable: false, formatter: controllerServiceActionFormatter, sortable: false, width: 90, maxWidth: 90}); @@@ -812,14 -793,14 +832,14 @@@ }, nf.Common.config.tooltipConfig)); } } -- ++ var bulletinIcon = $(this).find('img.has-bulletins'); if (bulletinIcon.length && !bulletinIcon.data('qtip')) { var taskId = $(this).find('span.row-id').text(); // get the task item var item = controllerServicesData.getItemById(taskId); -- ++ // format the tooltip var bulletins = nf.Common.getFormattedBulletins(item.bulletins); var tooltip = nf.Common.formatUnorderedList(bulletins); @@@ -948,6 -929,12 +968,12 @@@ * Hides the selected reporting task. */ var clearSelectedReportingTask = function () { + if (nf.Canvas.isClustered()) { + $('#reporting-task-availability-combo').combo('setSelectedOption', { + value: config.node + }); + } - ++ $('#reporting-task-type-description').text(''); $('#reporting-task-type-name').text(''); $('#selected-reporting-task-name').text(''); @@@ -969,7 -956,7 +995,7 @@@ /** * Performs the filtering. -- * ++ * * @param {object} item The item subject to filtering * @param {object} args Filter arguments * @returns {Boolean} Whether or not to include the item @@@ -1015,10 -1002,10 +1041,10 @@@ addReportingTask(selectedTaskType); } }; -- ++ /** * Adds a new reporting task of the specified type. -- * ++ * * @param {string} reportingTaskType */ var addReportingTask = function (reportingTaskType) { @@@ -1276,24 -1256,28 +1302,28 @@@ var moreReportingTaskDetails = function (row, cell, value, columnDef, dataContext) { var markup = '<img src="images/iconDetails.png" title="View Details" class="pointer view-reporting-task" style="margin-top: 5px; float: left;" />'; + + // always include a button to view the usage + markup += '<img src="images/iconUsage.png" title="Usage" class="pointer reporting-task-usage" style="margin-left: 6px; margin-top: 3px;"/>'; + var hasErrors = !nf.Common.isEmpty(dataContext.validationErrors); var hasBulletins = !nf.Common.isEmpty(dataContext.bulletins); -- ++ if (hasErrors) { - markup += '<img src="images/iconAlert.png" class="has-errors" style="margin-top: 4px; margin-left: 1px; float: left;" />'; + markup += '<img src="images/iconAlert.png" class="has-errors" style="margin-top: 4px; margin-left: 3px; float: left;" />'; } -- ++ if (hasBulletins) { markup += '<img src="images/iconBulletin.png" class="has-bulletins" style="margin-top: 5px; margin-left: 5px; float: left;"/>'; } -- ++ if (hasErrors || hasBulletins) { markup += '<span class="hidden row-id">' + nf.Common.escapeHtml(dataContext.id) + '</span>'; } -- ++ return markup; }; -- ++ var reportingTaskRunStatusFormatter = function (row, cell, value, columnDef, dataContext) { // determine the appropriate label var label; @@@ -1308,34 -1292,38 +1338,38 @@@ label = 'Disabled'; } } -- ++ // include the active thread count if appropriate var activeThreadCount = ''; if (nf.Common.isDefinedAndNotNull(dataContext.activeThreadCount) && dataContext.activeThreadCount > 0) { activeThreadCount = '(' + dataContext.activeThreadCount + ')'; } -- ++ // format the markup var formattedValue = '<div class="' + nf.Common.escapeHtml(label.toLowerCase()) + '" style="margin-top: 3px;"></div>'; return formattedValue + '<div class="status-text" style="margin-top: 2px; margin-left: 4px; float: left;">' + nf.Common.escapeHtml(label) + '</div><div style="float: left; margin-left: 4px;">' + nf.Common.escapeHtml(activeThreadCount) + '</div>'; }; -- ++ var reportingTaskActionFormatter = function (row, cell, value, columnDef, dataContext) { var markup = ''; - // only DFMs can edit reporting tasks + // only DFMs can edit reporting tasks and view state if (nf.Common.isDFM()) { if (dataContext.state === 'RUNNING') { - markup += '<img src="images/iconStop.png" title="Stop" class="pointer stop-reporting-task" style="margin-top: 2px;" /> '; + markup += '<img src="images/iconStop.png" title="Stop" class="pointer stop-reporting-task" style="margin-top: 2px;" />'; } else if (dataContext.state === 'STOPPED' || dataContext.state === 'DISABLED') { - markup += '<img src="images/iconEdit.png" title="Edit" class="pointer edit-reporting-task" style="margin-top: 2px;" /> '; - + markup += '<img src="images/iconEdit.png" title="Edit" class="pointer edit-reporting-task" style="margin-top: 2px;" />'; - ++ // support starting when stopped and no validation errors if (dataContext.state === 'STOPPED' && nf.Common.isEmpty(dataContext.validationErrors)) { - markup += '<img src="images/iconRun.png" title="Start" class="pointer start-reporting-task" style="margin-top: 2px;"/> '; + markup += '<img src="images/iconRun.png" title="Start" class="pointer start-reporting-task" style="margin-top: 2px; margin-left: 3px;"/>'; } -- - markup += '<img src="images/iconDelete.png" title="Remove" class="pointer delete-reporting-task" style="margin-top: 2px;" /> '; ++ + markup += '<img src="images/iconDelete.png" title="Remove" class="pointer delete-reporting-task" style="margin-top: 2px; margin-left: 3px;" />'; + } + + if (dataContext.persistsState === true) { + markup += '<img src="images/iconViewState.png" title="View State" class="pointer view-state-reporting-task" style="margin-top: 2px; margin-left: 3px;" />'; } } @@@ -1357,7 -1343,7 +1391,7 @@@ if (nf.Canvas.isClustered()) { reportingTasksColumnModel.push({id: 'availability', field: 'availability', name: 'Availability', formatter: availabilityFormatter, sortable: true, resizeable: true}); } -- ++ // action column should always be last reportingTasksColumnModel.push({id: 'actions', name: ' ', resizable: false, formatter: reportingTaskActionFormatter, sortable: false, width: 90, maxWidth: 90}); @@@ -1459,14 -1448,14 +1500,14 @@@ }, nf.Common.config.tooltipConfig)); } } -- ++ var bulletinIcon = $(this).find('img.has-bulletins'); if (bulletinIcon.length && !bulletinIcon.data('qtip')) { var taskId = $(this).find('span.row-id').text(); // get the task item var item = reportingTasksData.getItemById(taskId); -- ++ // format the tooltip var bulletins = nf.Common.getFormattedBulletins(item.bulletins); var tooltip = nf.Common.formatUnorderedList(bulletins); @@@ -1611,26 -1600,26 +1652,26 @@@ var selectedTab = $('li.settings-selected-tab').text(); if (selectedTab === 'Controller Services') { $('#new-controller-service-dialog').modal('show'); -- ++ // reset the canvas size after the dialog is shown var controllerServiceTypesGrid = $('#controller-service-types-table').data('gridInstance'); if (nf.Common.isDefinedAndNotNull(controllerServiceTypesGrid)) { controllerServiceTypesGrid.setSelectedRows([0]); controllerServiceTypesGrid.resizeCanvas(); } -- ++ // set the initial focus $('#controller-service-type-filter').focus(); } else if (selectedTab === 'Reporting Tasks') { $('#new-reporting-task-dialog').modal('show'); -- ++ // reset the canvas size after the dialog is shown var reportingTaskTypesGrid = $('#reporting-task-types-table').data('gridInstance'); if (nf.Common.isDefinedAndNotNull(reportingTaskTypesGrid)) { reportingTaskTypesGrid.setSelectedRows([0]); reportingTaskTypesGrid.resizeCanvas(); } -- ++ // set the initial focus $('#reporting-task-type-filter').focus(); } @@@ -1641,7 -1630,7 +1682,7 @@@ initControllerServices(); initReportingTasks(); }, -- ++ /** * Update the size of the grid based on its container's current size. */ @@@ -1656,7 -1645,7 +1697,7 @@@ reportingTasksGrid.resizeCanvas(); } }, -- ++ /** * Shows the settings dialog. */ @@@ -1666,11 -1655,11 +1707,11 @@@ // reset button state $('#settings-save').mouseout(); }); -- ++ // adjust the table size nf.Settings.resetTableSize(); }, -- ++ /** * Loads the settings. */ @@@ -1715,10 -1704,10 +1756,10 @@@ } }).fail(nf.Common.handleAjaxError); }, -- ++ /** * Sets the controller service and reporting task bulletins in their respective tables. -- * ++ * * @param {object} controllerServiceBulletins * @param {object} reportingTaskBulletins */ @@@ -1733,7 -1722,7 +1774,7 @@@ var controllerServiceBulletinsBySource = d3.nest() .key(function(d) { return d.sourceId; }) .map(controllerServiceBulletins, d3.map); -- ++ controllerServiceBulletinsBySource.forEach(function(sourceId, sourceBulletins) { var controllerService = controllerServicesData.getItemById(sourceId); if (nf.Common.isDefinedAndNotNull(controllerService)) { @@@ -1757,7 -1746,7 +1798,7 @@@ var reportingTasksGrid = $('#reporting-tasks-table').data('gridInstance'); var reportingTasksData = reportingTasksGrid.getData(); reportingTasksData.beginUpdate(); -- ++ // if there are some bulletins process them if (!nf.Common.isEmpty(reportingTaskBulletins)) { var reportingTaskBulletinsBySource = d3.nest() http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeGroovy.java ---------------------------------------------------------------------- diff --cc nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeGroovy.java index 349b25d,0000000..2dc700d mode 100644,000000..100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeGroovy.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/test/java/org/apache/nifi/processors/script/TestInvokeGroovy.java @@@ -1,171 -1,0 +1,173 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.script; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.MockProcessContext; +import org.apache.nifi.util.MockProcessorInitializationContext; +import org.apache.nifi.util.MockValidationContext; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.Before; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Set; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +public class TestInvokeGroovy extends BaseScriptTest { + + @Before + public void setup() throws Exception { + super.setupInvokeScriptProcessor(); + } + + /** + * Tests a script that has a Groovy Processor that that reads the first line of text from the flowfiles content and stores the value in an attribute of the outgoing flowfile. + * + * @throws Exception Any error encountered while testing + */ + @Test + public void testReadFlowFileContentAndStoreInFlowFileAttribute() throws Exception { + runner.setValidateExpressionUsage(false); + runner.setProperty(InvokeScriptedProcessor.SCRIPT_ENGINE, "Groovy"); + runner.setProperty(InvokeScriptedProcessor.SCRIPT_FILE, "target/test/resources/groovy/test_reader.groovy"); + runner.setProperty(InvokeScriptedProcessor.MODULES, "target/test/resources/groovy"); + + runner.assertValid(); + runner.enqueue("test content".getBytes(StandardCharsets.UTF_8)); + runner.run(); + + runner.assertAllFlowFilesTransferred("success", 1); + final List<MockFlowFile> result = runner.getFlowFilesForRelationship("success"); + result.get(0).assertAttributeEquals("from-content", "test content"); + } + + /** + * Tests a script that has a Groovy Processor that that reads the first line of text from the flowfiles content and + * stores the value in an attribute of the outgoing flowfile. + * + * @throws Exception Any error encountered while testing + */ + @Test + public void testScriptDefinedAttribute() throws Exception { + InvokeScriptedProcessor processor = new InvokeScriptedProcessor(); + MockProcessContext context = new MockProcessContext(processor); + MockProcessorInitializationContext initContext = new MockProcessorInitializationContext(processor, context); + + processor.initialize(initContext); + + context.setProperty(InvokeScriptedProcessor.SCRIPT_ENGINE, "Groovy"); + context.setProperty(InvokeScriptedProcessor.SCRIPT_FILE, "target/test/resources/groovy/test_reader.groovy"); + context.setProperty(InvokeScriptedProcessor.MODULES, "target/test/resources/groovy"); - processor.customValidate(new MockValidationContext(context)); ++ // State Manger is unused, and a null reference is specified ++ processor.customValidate(new MockValidationContext(context, null)); + processor.setup(context); + + List<PropertyDescriptor> descriptors = processor.getSupportedPropertyDescriptors(); + assertNotNull(descriptors); + assertTrue(descriptors.size() > 0); + boolean found = false; + for (PropertyDescriptor descriptor : descriptors) { + if (descriptor.getName().equals("test-attribute")) { + found = true; + break; + } + } + assertTrue(found); + } + + /** + * Tests a script that has a Groovy Processor that that reads the first line of text from the flowfiles content and + * stores the value in an attribute of the outgoing flowfile. + * + * @throws Exception Any error encountered while testing + */ + @Test + public void testScriptDefinedRelationship() throws Exception { + InvokeScriptedProcessor processor = new InvokeScriptedProcessor(); + MockProcessContext context = new MockProcessContext(processor); + MockProcessorInitializationContext initContext = new MockProcessorInitializationContext(processor, context); + + processor.initialize(initContext); + + context.setProperty(InvokeScriptedProcessor.SCRIPT_ENGINE, "Groovy"); + context.setProperty(InvokeScriptedProcessor.SCRIPT_FILE, "target/test/resources/groovy/test_reader.groovy"); - processor.customValidate(new MockValidationContext(context)); ++ // State Manger is unused, and a null reference is specified ++ processor.customValidate(new MockValidationContext(context, null)); + processor.setup(context); + + Set<Relationship> relationships = processor.getRelationships(); + assertNotNull(relationships); + assertTrue(relationships.size() > 0); + boolean found = false; + for (Relationship relationship : relationships) { + if (relationship.getName().equals("test")) { + found = true; + break; + } + } + assertTrue(found); + } + + /** + * Tests a script that throws a ProcessException within. The expected result is that the exception will be + * propagated + * + * @throws Exception Any error encountered while testing + */ + @Test(expected = AssertionError.class) + public void testInvokeScriptCausesException() throws Exception { + final TestRunner runner = TestRunners.newTestRunner(new InvokeScriptedProcessor()); + runner.setValidateExpressionUsage(false); + runner.setProperty(InvokeScriptedProcessor.SCRIPT_ENGINE, "Groovy"); + runner.setProperty(ExecuteScript.SCRIPT_BODY, getFileContentsAsString( + TEST_RESOURCE_LOCATION + "groovy/testInvokeScriptCausesException.groovy") + ); + runner.assertValid(); + runner.enqueue("test content".getBytes(StandardCharsets.UTF_8)); + runner.run(); + + } + + /** + * Tests a script that routes the FlowFile to failure. + * + * @throws Exception Any error encountered while testing + */ + @Test + public void testScriptRoutesToFailure() throws Exception { + runner.setValidateExpressionUsage(false); + runner.setProperty(InvokeScriptedProcessor.SCRIPT_ENGINE, "Groovy"); + runner.setProperty(ExecuteScript.SCRIPT_BODY, getFileContentsAsString( + TEST_RESOURCE_LOCATION + "groovy/testScriptRoutesToFailure.groovy") + ); + runner.assertValid(); + runner.enqueue("test content".getBytes(StandardCharsets.UTF_8)); + runner.run(); + + runner.assertAllFlowFilesTransferred(InvokeScriptedProcessor.REL_FAILURE, 1); + final List<MockFlowFile> result = runner.getFlowFilesForRelationship(InvokeScriptedProcessor.REL_FAILURE); + assertFalse(result.isEmpty()); + } +} http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java ---------------------------------------------------------------------- diff --cc nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java index 079f256,818cd3b..999bead --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java @@@ -99,9 -96,10 +96,11 @@@ import org.apache.nifi.util.Tuple @Tags({"get", "fetch", "poll", "http", "https", "ingest", "source", "input"}) @InputRequirement(Requirement.INPUT_FORBIDDEN) -@CapabilityDescription("Fetches a file via HTTP. If the HTTP server supports it, the Processor then stores the Last Modified time and the ETag " - + "so that data will not be pulled again until the remote data changes or until the state is cleared. Note that due to limitations on state " +@CapabilityDescription("Fetches data from an HTTP or HTTPS URL and writes the data to the content of a FlowFile. Once the content has been fetched, the ETag and Last Modified " - + "dates are remembered (if the web server supports these concepts). This allows the Processor to fetch new data only if the remote data has changed. That is, once the " - + "content has been fetched from the given URL, it will not be fetched again until the content on the remote server changes.") ++ + "dates are remembered (if the web server supports these concepts). This allows the Processor to fetch new data only if the remote data has changed or until the state is cleared. That is, " ++ + "once the content has been fetched from the given URL, it will not be fetched again until the content on the remote server changes. Note that due to limitations on state " + + "management, stored \"last modified\" and etag fields never expire. If the URL in GetHttp uses Expression Language that is unbounded, there " + + "is the potential for Out of Memory Errors to occur.") @WritesAttributes({ @WritesAttribute(attribute = "filename", description = "The filename is set to the name of the file on the remote server"), @WritesAttribute(attribute = "mime.type", description = "The MIME Type of the FlowFile, as reported by the HTTP Content-Type header") http://git-wip-us.apache.org/repos/asf/nifi/blob/4df65121/pom.xml ----------------------------------------------------------------------
