svn commit: r17155 - /dev/nifi/1.1.0/
Author: joewitt Date: Thu Nov 24 06:43:13 2016 New Revision: 17155 Log: NIFI-3100 RC1 artifacts Added: dev/nifi/1.1.0/ dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz (with props) dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.asc dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.md5 dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.sha1 dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.sha256 dev/nifi/1.1.0/nifi-1.1.0-bin.zip (with props) dev/nifi/1.1.0/nifi-1.1.0-bin.zip.asc dev/nifi/1.1.0/nifi-1.1.0-bin.zip.md5 dev/nifi/1.1.0/nifi-1.1.0-bin.zip.sha1 dev/nifi/1.1.0/nifi-1.1.0-bin.zip.sha256 dev/nifi/1.1.0/nifi-1.1.0-source-release.zip (with props) dev/nifi/1.1.0/nifi-1.1.0-source-release.zip.asc dev/nifi/1.1.0/nifi-1.1.0-source-release.zip.md5 dev/nifi/1.1.0/nifi-1.1.0-source-release.zip.sha1 dev/nifi/1.1.0/nifi-1.1.0-source-release.zip.sha256 dev/nifi/1.1.0/nifi-toolkit-1.1.0-bin.tar.gz (with props) dev/nifi/1.1.0/nifi-toolkit-1.1.0-bin.tar.gz.asc dev/nifi/1.1.0/nifi-toolkit-1.1.0-bin.tar.gz.md5 dev/nifi/1.1.0/nifi-toolkit-1.1.0-bin.tar.gz.sha1 dev/nifi/1.1.0/nifi-toolkit-1.1.0-bin.tar.gz.sha256 dev/nifi/1.1.0/nifi-toolkit-1.1.0-bin.zip (with props) dev/nifi/1.1.0/nifi-toolkit-1.1.0-bin.zip.asc dev/nifi/1.1.0/nifi-toolkit-1.1.0-bin.zip.md5 dev/nifi/1.1.0/nifi-toolkit-1.1.0-bin.zip.sha1 dev/nifi/1.1.0/nifi-toolkit-1.1.0-bin.zip.sha256 Added: dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz == Binary file - no diff available. Propchange: dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz -- svn:mime-type = application/octet-stream Added: dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.asc == --- dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.asc (added) +++ dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.asc Thu Nov 24 06:43:13 2016 @@ -0,0 +1,11 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v2 + +iQEcBAABCAAGBQJYNnm/AAoJEJCTv4VPgRoaqhgIAJD3DCbNioKgZqkZPFQRD4eg +JnJUYaHloxXJun9KuQu6qvR4MamAbSfOFM/opldt4P3jNAo6Aa/FPSkl46LZKJMF +RDQ5LLw+kdWiTGpHdTL+eMz9sPaJvyO5aXZD/Pu7t1T8z2vdAuR0HD8EAMxt6O1l +OUXbzYeaVje6nGx0E331Jzr4M6HZXXz7VYMpJWxorBtcunpXdDIzvU1y602KCjA3 +oFLdjj5UPDJYlvsvBraXw05r50ErQGAFvafZm+fi88r2IxLlD/cptElu4B0Xv+aO +r9p9HVGtCZespp+3IP1iM78xLZLLTnvR0Xrl+/8YKwSsNDy1wQIwRS807QuL6gU= +=Zz7I +-END PGP SIGNATURE- Added: dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.md5 == --- dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.md5 (added) +++ dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.md5 Thu Nov 24 06:43:13 2016 @@ -0,0 +1 @@ +305e06fd5f44853fcb40eff6fa50dedf Added: dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.sha1 == --- dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.sha1 (added) +++ dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.sha1 Thu Nov 24 06:43:13 2016 @@ -0,0 +1 @@ +82ed2fb1a4d30fec5e4e4fe33d76d2cbbb46d4cf Added: dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.sha256 == --- dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.sha256 (added) +++ dev/nifi/1.1.0/nifi-1.1.0-bin.tar.gz.sha256 Thu Nov 24 06:43:13 2016 @@ -0,0 +1 @@ +1d9e5c975da8e223d11dbc70c51a2c6287f87a20088979c33602aad7fde5bada Added: dev/nifi/1.1.0/nifi-1.1.0-bin.zip == Binary file - no diff available. Propchange: dev/nifi/1.1.0/nifi-1.1.0-bin.zip -- svn:mime-type = application/octet-stream Added: dev/nifi/1.1.0/nifi-1.1.0-bin.zip.asc == --- dev/nifi/1.1.0/nifi-1.1.0-bin.zip.asc (added) +++ dev/nifi/1.1.0/nifi-1.1.0-bin.zip.asc Thu Nov 24 06:43:13 2016 @@ -0,0 +1,11 @@ +-BEGIN PGP SIGNATURE- +Version: GnuPG v2 + +iQEcBAABCAAGBQJYNnnRAAoJEJCTv4VPgRoaNdIIAK+BjoGdGq7LHmbRuvxKdQ6P +7WB+8hhweZ7CNXoVq/EY3kFy0Sqwe+kpZE2t2ahQbxt5GIQNb4O9+ElCWksJxlrz +SNY7ECf+4dVBJz/zSpR8duqJd2pcv5a2oCutfGhigcIXhj1c8hEMBF+kjwGoWn7g +7BZr2Te6igHj3aAfnHtYgLx4eLecSuF45gURXMmn7F7Fj6hgtNg+eFZWDU2720C1 +FimiuLfRdh9d7FujbVatrkq648Qqk+0LUf0Ivk7ZmoHpBwWuEj1NJHoR939MWoCw +iWJft9AmR687U/WAbOr8IfOdvWxV/bz3rRLNwwenuCJnElhnrBbsqFibypf+u+I= +=ksdW +-END PGP SIGNATURE- Added: dev/nifi/1.1.0/nifi-1.1.0-bin.zip.md5 == --- dev/nifi/1.1.0/nifi-1.1.0-bin.zip.md5 (added) +++ dev/nifi/1.1.0/nifi-1.1.0-bin.zip.md5 Thu Nov 24 06:43:13 2016 @@ -0,0 +1 @@ +962f05712baee4bdfaf0eee44e16fa91 Added: dev/nifi/1.1.0/nifi-1.1.0-bin.zip.sha1 == --- dev/nifi/1.1.0/nifi-1.1.0-bin.zip.sha1 (added) +++ dev/nifi/1.1.
[nifi] Git Push Summary
Repository: nifi Updated Tags: refs/tags/nifi-1.1.0-RC1 [created] 65870ef00
nifi git commit: NIFI-2655 Implemented logic check for missing -n or -l arguments. Added and updated unit tests. Cleaned up unnecessary and commented code. Removed ignored unit tests. Cleaned up comme
Repository: nifi
Updated Branches:
refs/heads/master 2c3714536 -> 1b2b9f136
NIFI-2655 Implemented logic check for missing -n or -l arguments.
Added and updated unit tests.
Cleaned up unnecessary and commented code.
Removed ignored unit tests.
Cleaned up commented code.
This closes #1262.
Signed-off-by: Andy LoPresto
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/1b2b9f13
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/1b2b9f13
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/1b2b9f13
Branch: refs/heads/master
Commit: 1b2b9f136df73a8b0d5d21419b6ebc2206f78393
Parents: 2c37145
Author: Andy LoPresto
Authored: Tue Nov 22 14:49:33 2016 -0800
Committer: Andy LoPresto
Committed: Wed Nov 23 15:12:05 2016 -0800
--
.../nifi/properties/ConfigEncryptionTool.groovy | 22 +---
.../properties/ConfigEncryptionToolTest.groovy | 127 +++
2 files changed, 46 insertions(+), 103 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/nifi/blob/1b2b9f13/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
--
diff --git
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
index 71166f0..5956f53 100644
---
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
+++
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
@@ -284,10 +284,9 @@ class ConfigEncryptionTool {
logger.info("(dest) flow.xml.gz:
\t\t\t\t\t${outputFlowXmlPath}")
}
-// TODO: Implement in NIFI-2655
-//if (!commandLine.hasOption(NIFI_PROPERTIES_ARG) &&
!commandLine.hasOption(LOGIN_IDENTITY_PROVIDERS_ARG)) {
-//printUsageAndThrow("One of '-n'/'--${NIFI_PROPERTIES_ARG}'
or '-l'/'--${LOGIN_IDENTITY_PROVIDERS_ARG}' must be provided",
ExitCode.INVALID_ARGS)
-//}
+if (!commandLine.hasOption(NIFI_PROPERTIES_ARG) &&
!commandLine.hasOption(LOGIN_IDENTITY_PROVIDERS_ARG) &&
!commandLine.hasOption(DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG)) {
+printUsageAndThrow("One or both of
'-n'/'--${NIFI_PROPERTIES_ARG}' or '-l'/'--${LOGIN_IDENTITY_PROVIDERS_ARG}'
must be provided unless '-x'/--'${DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG}' is
specified", ExitCode.INVALID_ARGS)
+}
if (commandLine.hasOption(MIGRATION_ARG)) {
migration = true
@@ -1037,21 +1036,6 @@ class ConfigEncryptionTool {
fileToWrite && ((!fileToWrite.exists() &&
fileToWrite.absoluteFile.parentFile.canWrite()) || (fileToWrite.exists() &&
fileToWrite.canWrite()))
}
-private static String determineDefaultBootstrapConfPath() {
-String niFiToolkitPath = System.getenv(NIFI_TOOLKIT_HOME) ?: ""
-"${niFiToolkitPath ? niFiToolkitPath + "/" : ""}conf/bootstrap.conf"
-}
-
-private static String determineDefaultNiFiPropertiesPath() {
-String niFiToolkitPath = System.getenv(NIFI_TOOLKIT_HOME) ?: ""
-"${niFiToolkitPath ? niFiToolkitPath + "/" : ""}conf/nifi.properties"
-}
-
-private static String determineDefaultLoginIdentityProvidersPath() {
-String niFiToolkitPath = System.getenv(NIFI_TOOLKIT_HOME) ?: ""
-"${niFiToolkitPath ? niFiToolkitPath + "/" :
""}conf/login-identity-providers.xml"
-}
-
private static String deriveKeyFromPassword(String password) {
password = password?.trim()
if (!password || password.length() < MIN_PASSWORD_LENGTH) {
http://git-wip-us.apache.org/repos/asf/nifi/blob/1b2b9f13/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
--
diff --git
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
index ae0b252..a5e4103 100644
---
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
+++
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
@@ -207,7 +207,7 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
// Act
flags.each { String arg ->
-tool.parse([arg, bootstrapPath] as String[])
+tool.parse([
[2/2] nifi git commit: NIFI-3024 Added key migration for sensitive processor properties contained in flow.xml.gz. (nifi.sensitive.props.key)
NIFI-3024 Added key migration for sensitive processor properties contained in flow.xml.gz. (nifi.sensitive.props.key) This closes #1261. Signed-off-by: Andy LoPresto Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/2c371453 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/2c371453 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/2c371453 Branch: refs/heads/master Commit: 2c3714536fc516fc1712dac2a7a9ccd855e6f25b Parents: cdb9b81 Author: Andy LoPresto Authored: Mon Nov 21 21:19:18 2016 -0800 Committer: Andy LoPresto Committed: Wed Nov 23 13:26:18 2016 -0800 -- .../src/main/asciidoc/administration-guide.adoc | 32 +- .../nifi/properties/NiFiPropertiesLoader.java | 64 +- .../properties/ProtectedNiFiProperties.java | 2 +- .../AESSensitivePropertyProviderTest.groovy | 2 +- .../nifi-toolkit-encrypt-config/pom.xml | 12 + .../nifi/properties/ConfigEncryptionTool.groovy | 483 - .../properties/ConfigEncryptionToolTest.groovy | 981 ++- .../src/test/resources/flow.xml | 154 +++ .../src/test/resources/flow.xml.gz | Bin 0 -> 1674 bytes .../src/test/resources/flow_default_key.xml | 154 +++ .../src/test/resources/flow_default_key.xml.gz | Bin 0 -> 1686 bytes .../src/test/resources/nifi_default.properties | 125 +++ ...erties_protected_aes_password_128.properties | 34 + 13 files changed, 1960 insertions(+), 83 deletions(-) -- http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-docs/src/main/asciidoc/administration-guide.adoc -- diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc index 2d63a9a..287760a 100644 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -1002,19 +1002,25 @@ The default encryption algorithm utilized is AES/GCM 128/256-bit. 128-bit is use You can use the following command line options with the `encrypt-config` tool: -* `-b,--bootstrapConf `The bootstrap.conf file to persist master key -* `-e,--oldKey ` The old raw hexadecimal key to use during key migration -* `-h,--help` Prints this usage message -* `-k,--key ` The raw hexadecimal key to use to encrypt the sensitive properties -* `-m,--migrate`If provided, the sensitive properties will be re-encrypted with a new key -* `-n,--niFiProperties ` The nifi.properties file containing unprotected config values (will be overwritten) -* `-o,--outputNiFiProperties ` The destination nifi.properties file containing protected config values (will not modify input nifi.properties) -* `-p,--password ` The password from which to derive the key to use to encrypt the sensitive properties -* `-r,--useRawKey` If provided, the secure console will prompt for the raw key value in hexadecimal form -* `-v,--verbose`Sets verbose mode (default false) -* `-w,--oldPassword ` The old password from which to derive the key during migration -* `-l,--loginIdentityProviders ` The login-identity-providers.xml file containing unprotected config values (will be overwritten) -* `-i,--outputLoginIdentityProviders ` The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml) + * `-A`,`--newFlowAlgorithm ` The algorithm to use to encrypt the sensitive processor properties in flow.xml.gz + * `-b`,`--bootstrapConf ` The bootstrap.conf file to persist master key + * `-e`,`--oldKey ` The old raw hexadecimal key to use during key migration + * `-f`,`--flowXml `The flow.xml.gz file currently protected with old password (will be overwritten) + * `-g`,`--outputFlowXml ` The destination flow.xml.gz file containing protected config values (will not modify input flow.xml.gz) + * `-h`,`--help` Prints this usage message + * `-i`,`--outputLoginIdentityProviders ` The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml) + * `-k`,`--key `The raw hexadecimal key to use to encrypt the sensitive properties + * `-l`,`--loginIdentityProviders ` The login-identity-providers.xml file containing unprotected config values (will be overwritten) + * `-m`,`--migrate` If provid
[1/2] nifi git commit: NIFI-3024 Added key migration for sensitive processor properties contained in flow.xml.gz. (nifi.sensitive.props.key)
Repository: nifi
Updated Branches:
refs/heads/master cdb9b81f4 -> 2c3714536
http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
--
diff --git
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
index 32a1975..ae0b252 100644
---
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
+++
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy
@@ -19,6 +19,7 @@ package org.apache.nifi.properties
import org.apache.commons.lang3.SystemUtils
import org.apache.log4j.AppenderSkeleton
import org.apache.log4j.spi.LoggingEvent
+import org.apache.nifi.encrypt.StringEncryptor
import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException
import org.apache.nifi.util.NiFiProperties
import org.apache.nifi.util.console.TextDevice
@@ -41,6 +42,10 @@ import org.slf4j.Logger
import org.slf4j.LoggerFactory
import javax.crypto.Cipher
+import javax.crypto.SecretKey
+import javax.crypto.SecretKeyFactory
+import javax.crypto.spec.PBEKeySpec
+import javax.crypto.spec.PBEParameterSpec
import java.nio.file.Files
import java.nio.file.attribute.PosixFilePermission
import java.security.KeyException
@@ -68,9 +73,20 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
private static
final String PASSWORD_KEY_HEX = isUnlimitedStrengthCryptoAvailable() ?
PASSWORD_KEY_HEX_256 : PASSWORD_KEY_HEX_128
+// Known issue documented in NIFI-1465 and NIFI-1255 where the password
cannot be > 16 characters without the JCE unlimited strength policies installed
+private static final String FLOW_PASSWORD_128 = "shortPassword"
+private static final String FLOW_PASSWORD_256 = "thisIsABadPassword"
+public static
+final String FLOW_PASSWORD = isUnlimitedStrengthCryptoAvailable() ?
FLOW_PASSWORD_256 : FLOW_PASSWORD_128
+
private static final int LIP_PASSWORD_LINE_COUNT = 3
private final String PASSWORD_PROP_REGEX = "]* name=\".*
Password\""
+private static final String DEFAULT_ALGORITHM =
"PBEWITHMD5AND256BITAES-CBC-OPENSSL"
+private static final String DEFAULT_PROVIDER = "BC"
+private static final String WFXCTR =
ConfigEncryptionTool.WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX
+private final String DEFAULT_LEGACY_SENSITIVE_PROPS_KEY = "nififtw!"
+
@BeforeClass
public static void setUpOnce() throws Exception {
Security.addProvider(new BouncyCastleProvider())
@@ -1269,7 +1285,14 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
assert
updatedLines.contains("${key}=${niFiProperties.getProperty(key)}".toString())
}
-assert originalLines == updatedLines
+if (originalLines.size() != updatedLines.size()) {
+// In situations where the original nifi.properties did not have a
protection scheme for nifi.sensitive.props.key, it is added automatically now
+def differentLines = updatedLines - originalLines
+assert differentLines.size() == 1
+assert differentLines.first() ==
"nifi.sensitive.props.key.protected="
+} else {
+assert originalLines == updatedLines
+}
logger.info("Updated nifi.properties:")
logger.info("\n" * 2 + updatedLines.join("\n"))
@@ -1304,7 +1327,14 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
assert
updatedLines.contains("${key}=${niFiProperties.getProperty(key)}".toString())
}
-assert originalLines == updatedLines
+if (originalLines.size() != updatedLines.size()) {
+// In situations where the original nifi.properties did not have a
protection scheme for nifi.sensitive.props.key, it is added automatically now
+def differentLines = updatedLines - originalLines
+assert differentLines.size() == 1
+assert differentLines.first() ==
"nifi.sensitive.props.key.protected="
+} else {
+assert originalLines == updatedLines
+}
logger.info("Updated nifi.properties:")
logger.info("\n" * 2 + updatedLines.join("\n"))
@@ -2308,7 +2338,9 @@ class ConfigEncryptionToolTest extends GroovyTestCase {
// Assert
assert serializedLines == encryptedLines
-assert TestAppender.events.any { it.renderedMessage =~ "No provider
element with class org.apache.nifi.ldap.LdapProvider found in XML content; the
file could be empty or the element may be missing or commented out" }
+assert TestAppender.events.any {
+it.renderedMessage =~ "No provider
nifi git commit: NIFI-3092 This closes #1268. Double quoting java invocation in bat files with %*
Repository: nifi Updated Branches: refs/heads/master 7ff14f719 -> cdb9b81f4 NIFI-3092 This closes #1268. Double quoting java invocation in bat files with %* Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/cdb9b81f Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/cdb9b81f Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/cdb9b81f Branch: refs/heads/master Commit: cdb9b81f4283052f6626a96e31e44c7da11d25a8 Parents: 7ff14f7 Author: Bryan Rosander Authored: Wed Nov 23 12:05:27 2016 -0500 Committer: joewitt Committed: Wed Nov 23 15:00:40 2016 -0500 -- .../src/main/resources/bin/encrypt-config.bat | 2 +- nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/s2s.bat | 2 +- .../nifi-toolkit-assembly/src/main/resources/bin/tls-toolkit.bat | 2 +- .../nifi-toolkit-assembly/src/main/resources/bin/zk-migrator.bat | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/nifi/blob/cdb9b81f/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat -- diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat index ca1ef07..a89e3fb 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/encrypt-config.bat @@ -35,6 +35,6 @@ set LIB_DIR=%~dp0..\classpath;%~dp0..\lib SET JAVA_PARAMS=-cp %LIB_DIR%\* -Xms128m -Xmx256m org.apache.nifi.properties.ConfigEncryptionTool -cmd.exe /C "%JAVA_EXE%" %JAVA_PARAMS% %* +cmd.exe /C ""%JAVA_EXE%" %JAVA_PARAMS% %* "" popd http://git-wip-us.apache.org/repos/asf/nifi/blob/cdb9b81f/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/s2s.bat -- diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/s2s.bat b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/s2s.bat index 6155715..7f0c0b2 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/s2s.bat +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/s2s.bat @@ -35,6 +35,6 @@ set LIB_DIR=%~dp0..\classpath;%~dp0..\lib SET JAVA_PARAMS=-cp %LIB_DIR%\* -Xms128m -Xmx256m %JAVA_ARGS% org.apache.nifi.toolkit.s2s.SiteToSiteCliMain -cmd.exe /C "%JAVA_EXE%" %JAVA_PARAMS% %* +cmd.exe /C ""%JAVA_EXE%" %JAVA_PARAMS% %* "" popd http://git-wip-us.apache.org/repos/asf/nifi/blob/cdb9b81f/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/tls-toolkit.bat -- diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/tls-toolkit.bat b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/tls-toolkit.bat index 025ec10..aaf7fe5 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/tls-toolkit.bat +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/tls-toolkit.bat @@ -35,6 +35,6 @@ set LIB_DIR=%~dp0..\classpath;%~dp0..\lib SET JAVA_PARAMS=-cp %LIB_DIR%\* -Xms12m -Xmx24m %JAVA_ARGS% org.apache.nifi.toolkit.tls.TlsToolkitMain -cmd.exe /C "%JAVA_EXE%" %JAVA_PARAMS% %* +cmd.exe /C ""%JAVA_EXE%" %JAVA_PARAMS% %* "" popd http://git-wip-us.apache.org/repos/asf/nifi/blob/cdb9b81f/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/zk-migrator.bat -- diff --git a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/zk-migrator.bat b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/zk-migrator.bat index 86bd480..5887ea7 100644 --- a/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/zk-migrator.bat +++ b/nifi-toolkit/nifi-toolkit-assembly/src/main/resources/bin/zk-migrator.bat @@ -35,6 +35,6 @@ set LIB_DIR=%~dp0..\classpath;%~dp0..\lib SET JAVA_PARAMS=-cp %LIB_DIR%\* -Xms12m -Xmx24m %JAVA_ARGS% org.apache.nifi.toolkit.zkmigrator.ZooKeeperMigratorMain -cmd.exe /C "%JAVA_EXE%" %JAVA_PARAMS% %* +cmd.exe /C ""%JAVA_EXE%" %JAVA_PARAMS% %* "" popd
nifi-site git commit: Adding myself to committers
Repository: nifi-site Updated Branches: refs/heads/master 43c3c6816 -> 328b5f9e2 Adding myself to committers Project: http://git-wip-us.apache.org/repos/asf/nifi-site/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi-site/commit/328b5f9e Tree: http://git-wip-us.apache.org/repos/asf/nifi-site/tree/328b5f9e Diff: http://git-wip-us.apache.org/repos/asf/nifi-site/diff/328b5f9e Branch: refs/heads/master Commit: 328b5f9e21e83227e29878352ef503a5674d56ea Parents: 43c3c68 Author: Scott Aslan Authored: Wed Nov 23 12:59:39 2016 -0500 Committer: Scott Aslan Committed: Wed Nov 23 12:59:39 2016 -0500 -- src/pages/html/people.hbs | 5 + 1 file changed, 5 insertions(+) -- http://git-wip-us.apache.org/repos/asf/nifi-site/blob/328b5f9e/src/pages/html/people.hbs -- diff --git a/src/pages/html/people.hbs b/src/pages/html/people.hbs index 24ea37d..1aa8ccd 100644 --- a/src/pages/html/people.hbs +++ b/src/pages/html/people.hbs @@ -178,6 +178,11 @@ title: Apache NiFi Team Bryan Rosander + +scottyaslan +Scott Aslan + +
svn commit: r1771011 - /nifi/site/trunk/people.html
Author: scottyaslan Date: Wed Nov 23 18:05:03 2016 New Revision: 1771011 URL: http://svn.apache.org/viewvc?rev=1771011&view=rev Log: Adding myself to committers Modified: nifi/site/trunk/people.html Modified: nifi/site/trunk/people.html URL: http://svn.apache.org/viewvc/nifi/site/trunk/people.html?rev=1771011&r1=1771010&r2=1771011&view=diff == --- nifi/site/trunk/people.html (original) +++ nifi/site/trunk/people.html Wed Nov 23 18:05:03 2016 @@ -275,6 +275,11 @@ Bryan Rosander + +scottyaslan +Scott Aslan + +
nifi git commit: NIFI-3091: Ensure that we set the appropriate size on FlowFiles when modifying them
Repository: nifi
Updated Branches:
refs/heads/master 91ff810db -> 7ff14f719
NIFI-3091: Ensure that we set the appropriate size on FlowFiles when modifying
them
This closes #1267
Signed-off-by: jpercivall
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/7ff14f71
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/7ff14f71
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/7ff14f71
Branch: refs/heads/master
Commit: 7ff14f7191f8bbb1722340ccdd963d3e7d24b9e4
Parents: 91ff810
Author: Mark Payne
Authored: Wed Nov 23 11:09:42 2016 -0500
Committer: jpercivall
Committed: Wed Nov 23 12:07:44 2016 -0500
--
.../repository/StandardProcessSession.java | 3 +-
.../repository/TestStandardProcessSession.java | 59
2 files changed, 61 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/nifi/blob/7ff14f71/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
--
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
index 002bac9..80c917c 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/StandardProcessSession.java
@@ -2580,7 +2580,8 @@ public final class StandardProcessSession implements
ProcessSession, ProvenanceE
cnfeThrown = true;
throw cnfe;
} finally {
-this.bytesWritten += countingOut.getBytesWritten();
+writtenToFlowFile = countingOut.getBytesWritten();
+this.bytesWritten += writtenToFlowFile;
this.bytesRead += countingIn.getBytesRead();
recursionSet.remove(source);
http://git-wip-us.apache.org/repos/asf/nifi/blob/7ff14f71/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
--
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
index 8cc088d..6f94994 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/repository/TestStandardProcessSession.java
@@ -277,6 +277,65 @@ public class TestStandardProcessSession {
}
@Test
+public void testModifyContentWithStreamCallbackHasCorrectSize() throws
IOException {
+final FlowFileRecord flowFileRecord = new
StandardFlowFileRecord.Builder()
+.id(1000L)
+.addAttribute("uuid", "12345678-1234-1234-1234-123456789012")
+.entryDate(System.currentTimeMillis())
+.build();
+flowFileQueue.put(flowFileRecord);
+FlowFile original = session.get();
+assertNotNull(original);
+
+FlowFile child = session.write(original, (in, out) ->
out.write("hello".getBytes()));
+session.transfer(child);
+session.commit();
+
+final FlowFileRecord onQueue =
flowFileQueue.poll(Collections.emptySet());
+assertEquals(5, onQueue.getSize());
+}
+
+@Test
+public void testModifyContentWithOutputStreamCallbackHasCorrectSize()
throws IOException {
+final FlowFileRecord flowFileRecord = new
StandardFlowFileRecord.Builder()
+.id(1000L)
+.addAttribute("uuid", "12345678-1234-1234-1234-123456789012")
+.entryDate(System.currentTimeMillis())
+.build();
+flowFileQueue.put(flowFileRecord);
+FlowFile original = session.get();
+assertNotNull(original);
+
+FlowFile child = session.write(original, out ->
out.write("hello".getBytes()));
+session.transfer(child);
+session.commit();
+
+fina
nifi git commit: NIFI-3088: Ensure that on recovery of FlowFile Repository, if we find a FlowFile that maps to an unknown queue that we log a warning that the queue is missing and drop the FlowFile, r
Repository: nifi
Updated Branches:
refs/heads/master 22143e386 -> 91ff810db
NIFI-3088: Ensure that on recovery of FlowFile Repository, if we find a
FlowFile that maps to an unknown queue that we log a warning that the queue is
missing and drop the FlowFile, rather than throwing an NPE
This closes #1266
Signed-off-by: jpercivall
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/91ff810d
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/91ff810d
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/91ff810d
Branch: refs/heads/master
Commit: 91ff810dbae71ac9d49bb5bcb0ef6b7c8dac8036
Parents: 22143e3
Author: Mark Payne
Authored: Wed Nov 23 10:33:06 2016 -0500
Committer: jpercivall
Committed: Wed Nov 23 11:27:14 2016 -0500
--
.../repository/SchemaRepositoryRecordSerde.java | 20 +++-
.../WriteAheadFlowFileRepository.java | 10 --
.../schema/RepositoryRecordFieldMap.java| 2 +-
.../schema/RepositoryRecordSchema.java | 1 +
4 files changed, 29 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/nifi/blob/91ff810d/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/SchemaRepositoryRecordSerde.java
--
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/SchemaRepositoryRecordSerde.java
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/SchemaRepositoryRecordSerde.java
index 916fd76..c0c9d18 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/SchemaRepositoryRecordSerde.java
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/SchemaRepositoryRecordSerde.java
@@ -38,10 +38,13 @@ import org.apache.nifi.repository.schema.Repetition;
import org.apache.nifi.repository.schema.SchemaRecordReader;
import org.apache.nifi.repository.schema.SchemaRecordWriter;
import org.apache.nifi.repository.schema.SimpleRecordField;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.wali.SerDe;
import org.wali.UpdateType;
public class SchemaRepositoryRecordSerde extends RepositoryRecordSerde
implements SerDe {
+private static final Logger logger =
LoggerFactory.getLogger(SchemaRepositoryRecordSerde.class);
private static final int MAX_ENCODING_VERSION = 1;
private final RecordSchema writeSchema =
RepositoryRecordSchema.REPOSITORY_RECORD_SCHEMA_V1;
@@ -154,7 +157,19 @@ public class SchemaRepositoryRecordSerde extends
RepositoryRecordSerde implement
final String queueId = (String)
record.getFieldValue(RepositoryRecordSchema.QUEUE_IDENTIFIER);
final FlowFileQueue queue = getFlowFileQueue(queueId);
-return new StandardRepositoryRecord(queue, flowFileRecord);
+final StandardRepositoryRecord repoRecord = new
StandardRepositoryRecord(queue, flowFileRecord);
+requireFlowFileQueue(repoRecord, queueId);
+return repoRecord;
+}
+
+private void requireFlowFileQueue(final StandardRepositoryRecord
repoRecord, final String queueId) {
+if (queueId == null || queueId.trim().isEmpty()) {
+logger.warn("{} does not have a Queue associated with it; this
record will be discarded", repoRecord.getCurrent());
+repoRecord.markForAbort();
+} else if (repoRecord.getOriginalQueue() == null) {
+logger.warn("{} maps to unknown Queue {}; this record will be
discarded", repoRecord.getCurrent(), queueId);
+repoRecord.markForAbort();
+}
}
private void populateContentClaim(final StandardFlowFileRecord.Builder
ffBuilder, final Record record) {
@@ -189,6 +204,9 @@ public class SchemaRepositoryRecordSerde extends
RepositoryRecordSerde implement
final StandardRepositoryRecord repoRecord = createRecord(record);
final String swapLocation = (String) record.getFieldValue(new
SimpleRecordField(RepositoryRecordSchema.SWAP_LOCATION, FieldType.STRING,
Repetition.EXACTLY_ONE));
repoRecord.setSwapLocation(swapLocation);
+
+final String queueId = (String)
record.getFieldValue(RepositoryRecordSchema.QUEUE_IDENTIFIER);
+requireFlowFileQueue(repoRecord, queueId);
return repoRecord;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/91ff810d/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/WriteAheadFlo
nifi git commit: NIFI-2991 This closes #1265. was allowed to restore twitter4j json.org usage until April. Resolution for that will be in NIFI-3089
Repository: nifi Updated Branches: refs/heads/master 7fc7494b2 -> 22143e386 NIFI-2991 This closes #1265. was allowed to restore twitter4j json.org usage until April. Resolution for that will be in NIFI-3089 Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/22143e38 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/22143e38 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/22143e38 Branch: refs/heads/master Commit: 22143e386d07761ca92b7cb889ec8c11d7d130e9 Parents: 7fc7494 Author: joewitt Authored: Wed Nov 23 10:20:18 2016 -0500 Committer: joewitt Committed: Wed Nov 23 10:21:41 2016 -0500 -- nifi-assembly/NOTICE| 28 nifi-assembly/pom.xml | 2 -- .../nifi-social-media-nar/pom.xml | 15 --- .../src/main/resources/META-INF/NOTICE | 26 -- 4 files changed, 52 insertions(+), 19 deletions(-) -- http://git-wip-us.apache.org/repos/asf/nifi/blob/22143e38/nifi-assembly/NOTICE -- diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE index 4a93017..5711c65 100644 --- a/nifi-assembly/NOTICE +++ b/nifi-assembly/NOTICE @@ -132,6 +132,34 @@ The following binary components are provided under the Apache Software License v Apache Commons Logging Copyright 2003-2014 The Apache Software Foundation + (ASLv2) Twitter4J +The following NOTICE information applies: + Copyright 2007 Yusuke Yamamoto + + Twitter4J includes software from JSON.org to parse JSON response from the Twitter API. + You can see the license term at http://www.JSON.org/license.html + + Copyright (c) 2002 JSON.org + + Permission is hereby granted, free of charge, to any person obtaining a copy of this + software and associated documentation files (the "Software"), to deal in the Software + without restriction, including without limitation the rights to use, copy, modify, + merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + The Software shall be used for Good, not Evil. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + (ASLv2) Apache Commons VFS The following NOTICE information applies: Apache Commons VFS http://git-wip-us.apache.org/repos/asf/nifi/blob/22143e38/nifi-assembly/pom.xml -- diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml index 0ac6225..8f6fa55 100755 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -243,13 +243,11 @@ language governing permissions and limitations under the License. --> nifi-solr-nar nar - org.apache.nifi nifi-hl7-nar http://git-wip-us.apache.org/repos/asf/nifi/blob/22143e38/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/pom.xml -- diff --git a/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/pom.xml b/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/pom.xml index 9eb3391..d6f13b0 100644 --- a/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/pom.xml +++ b/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/pom.xml @@ -36,19 +36,4 @@ 1.1.0-SNAPSHOT - - - - -org.apache.maven.plugins -maven-deploy-plugin - -true - - - - http://git-wip-us.apache.org/repos/asf/nifi/blob/22143e38/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/src/main/resources/META-INF/NOTICE -- diff --git a/nifi-nar-bundles/nifi-social-media-bundle/nifi-social-media-nar/src/main/resources/META-INF/NOTICE b/nifi-nar-bundles/nifi-social-media-bundle/nifi-socia
[jira] [Commented] (MINIFI-114) nifi-minifi-cpp default "State Location" value "Local" causes error
[ https://issues.apache.org/jira/browse/MINIFI-114?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15690201#comment-15690201 ] ASF GitHub Bot commented on MINIFI-114: --- Github user apiri commented on the issue: https://github.com/apache/nifi-minifi-cpp/pull/24 reviewing > nifi-minifi-cpp default "State Location" value "Local" causes error > --- > > Key: MINIFI-114 > URL: https://issues.apache.org/jira/browse/MINIFI-114 > Project: Apache NiFi MiNiFi > Issue Type: Bug > Components: C++, Core Framework >Reporter: Randy Gelhausen >Assignee: bqiu > Fix For: cpp-0.1.0 > > > Example config: > - name: TailFile > class: org.apache.nifi.processors.standard.TailFile > max concurrent tasks: 1 > scheduling strategy: TIMER_DRIVEN > scheduling period: 0 sec > penalization period: 30 sec > yield period: 1 sec > run duration nanos: 0 > auto-terminated relationships list: [] > Properties: > File Location: Local > File to Tail: /var/log/anaconda/syslog > Initial Start Position: Beginning of File > Rolling Filename Pattern: > [2016-10-03 17:05:50.273] [minifi log] [error] store state file failed -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[jira] [Updated] (MINIFI-114) nifi-minifi-cpp default "State Location" value "Local" causes error
[ https://issues.apache.org/jira/browse/MINIFI-114?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ] Aldrin Piri updated MINIFI-114: --- Assignee: bqiu > nifi-minifi-cpp default "State Location" value "Local" causes error > --- > > Key: MINIFI-114 > URL: https://issues.apache.org/jira/browse/MINIFI-114 > Project: Apache NiFi MiNiFi > Issue Type: Bug > Components: C++, Core Framework >Reporter: Randy Gelhausen >Assignee: bqiu > Fix For: cpp-0.1.0 > > > Example config: > - name: TailFile > class: org.apache.nifi.processors.standard.TailFile > max concurrent tasks: 1 > scheduling strategy: TIMER_DRIVEN > scheduling period: 0 sec > penalization period: 30 sec > yield period: 1 sec > run duration nanos: 0 > auto-terminated relationships list: [] > Properties: > File Location: Local > File to Tail: /var/log/anaconda/syslog > Initial Start Position: Beginning of File > Rolling Filename Pattern: > [2016-10-03 17:05:50.273] [minifi log] [error] store state file failed -- This message was sent by Atlassian JIRA (v6.3.4#6332)
[1/2] nifi git commit: NIFI-3087: Fixed issue with partial failure responses in PutElasticsearch(Http)
Repository: nifi
Updated Branches:
refs/heads/master 066accc27 -> 7fc7494b2
NIFI-3087: Fixed issue with partial failure responses in PutElasticsearch(Http)
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/d3dbac50
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/d3dbac50
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/d3dbac50
Branch: refs/heads/master
Commit: d3dbac50a8f354503838e5a0bdf22872d878b078
Parents: 066accc
Author: Matt Burgess
Authored: Tue Nov 22 20:08:17 2016 -0500
Committer: joewitt
Committed: Wed Nov 23 08:11:49 2016 -0500
--
.../elasticsearch/PutElasticsearch.java | 24
.../elasticsearch/PutElasticsearchHttp.java | 2 +-
2 files changed, 15 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3dbac50/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch.java
--
diff --git
a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch.java
b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch.java
index f64180b..216efd4 100644
---
a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch.java
+++
b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearch.java
@@ -211,17 +211,21 @@ public class PutElasticsearch extends
AbstractElasticsearchTransportClientProces
final BulkResponse response = bulk.execute().actionGet();
if (response.hasFailures()) {
-for (final BulkItemResponse item : response.getItems()) {
-final FlowFile flowFile =
flowFilesToTransfer.get(item.getItemId());
-if (item.isFailed()) {
-logger.error("Failed to insert {} into Elasticsearch
due to {}, transferring to failure",
-new Object[]{flowFile,
item.getFailure().getMessage()});
-session.transfer(flowFile, REL_FAILURE);
-
-} else {
-session.transfer(flowFile, REL_SUCCESS);
+// Responses are guaranteed to be in order, remove them in
reverse order
+BulkItemResponse[] responses = response.getItems();
+if (responses != null && responses.length > 0) {
+for (int i = responses.length - 1; i >= 0; i--) {
+final FlowFile flowFile = flowFilesToTransfer.get(i);
+if (responses[i].isFailed()) {
+logger.error("Failed to insert {} into
Elasticsearch due to {}, transferring to failure",
+new Object[]{flowFile,
responses[i].getFailure().getMessage()});
+session.transfer(flowFile, REL_FAILURE);
+
+} else {
+session.transfer(flowFile, REL_SUCCESS);
+}
+flowFilesToTransfer.remove(flowFile);
}
-flowFilesToTransfer.remove(flowFile);
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/d3dbac50/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchHttp.java
--
diff --git
a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchHttp.java
b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchHttp.java
index 7117100..3ba46bb 100644
---
a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchHttp.java
+++
b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/main/java/org/apache/nifi/processors/elasticsearch/PutElasticsearchHttp.java
@@ -328,7 +328,7 @@ public class PutElasticsearchHttp extends
AbstractElasticsearchHttpProcessor {
if (itemNodeArray.size() > 0) {
// All items are returned whether they succeeded
or failed, so iterate through the item array
[2/2] nifi git commit: NIFI-3087: This closes #1263. Added unit tests to PutElasticsearch(Http) to illustrate issue
NIFI-3087: This closes #1263. Added unit tests to PutElasticsearch(Http) to
illustrate issue
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/7fc7494b
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/7fc7494b
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/7fc7494b
Branch: refs/heads/master
Commit: 7fc7494b2174340a3caa0292b4a54859eaf16a34
Parents: d3dbac5
Author: Matt Burgess
Authored: Tue Nov 22 20:07:31 2016 -0500
Committer: joewitt
Committed: Wed Nov 23 08:22:34 2016 -0500
--
.../elasticsearch/TestPutElasticsearch.java | 22 ++--
.../elasticsearch/TestPutElasticsearchHttp.java | 16 +++---
2 files changed, 25 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/nifi/blob/7fc7494b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestPutElasticsearch.java
--
diff --git
a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestPutElasticsearch.java
b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestPutElasticsearch.java
index d7fb439..4e6a820 100644
---
a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestPutElasticsearch.java
+++
b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestPutElasticsearch.java
@@ -112,15 +112,19 @@ public class TestPutElasticsearch {
runner.setProperty(AbstractElasticsearchTransportClientProcessor.SAMPLER_INTERVAL,
"5s");
runner.setProperty(PutElasticsearch.INDEX, "doc");
runner.setProperty(PutElasticsearch.TYPE, "status");
-runner.setProperty(PutElasticsearch.BATCH_SIZE, "1");
+runner.setProperty(PutElasticsearch.BATCH_SIZE, "2");
runner.setProperty(PutElasticsearch.ID_ATTRIBUTE, "doc_id");
runner.enqueue(docExample, new HashMap() {{
put("doc_id", "28039652140");
}});
+runner.enqueue(docExample, new HashMap() {{
+put("doc_id", "28039652141");
+}});
runner.run(1, true, true);
-runner.assertAllFlowFilesTransferred(PutElasticsearch.REL_FAILURE, 1);
+runner.assertTransferCount(PutElasticsearch.REL_FAILURE, 1);
+runner.assertTransferCount(PutElasticsearch.REL_SUCCESS, 1);
final MockFlowFile out =
runner.getFlowFilesForRelationship(PutElasticsearch.REL_FAILURE).get(0);
assertNotNull(out);
out.assertAttributeEquals("doc_id", "28039652140");
@@ -349,10 +353,16 @@ public class TestPutElasticsearch {
public BulkResponse get() throws InterruptedException,
ExecutionException {
BulkResponse response = mock(BulkResponse.class);
when(response.hasFailures()).thenReturn(responseHasFailures);
-BulkItemResponse item = mock(BulkItemResponse.class);
-when(item.getItemId()).thenReturn(1);
-when(item.isFailed()).thenReturn(true);
-when(response.getItems()).thenReturn(new
BulkItemResponse[]{item});
+BulkItemResponse item1 = mock(BulkItemResponse.class);
+BulkItemResponse item2 = mock(BulkItemResponse.class);
+when(item1.getItemId()).thenReturn(1);
+when(item1.isFailed()).thenReturn(true);
+BulkItemResponse.Failure failure =
mock(BulkItemResponse.Failure.class);
+when(failure.getMessage()).thenReturn("Bad message");
+when(item1.getFailure()).thenReturn(failure);
+when(item2.getItemId()).thenReturn(2);
+when(item2.isFailed()).thenReturn(false);
+when(response.getItems()).thenReturn(new
BulkItemResponse[]{item1, item2});
return response;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/7fc7494b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestPutElasticsearchHttp.java
--
diff --git
a/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestPutElasticsearchHttp.java
b/nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-processors/src/test/java/org/apache/nifi/processors/elasticsearch/TestPutElasticsearchHttp.java
index 1172004..9ce578f 100644
---
a/nifi-nar-
