Repository: nifi Updated Branches: refs/heads/master cdb9b81f4 -> 2c3714536
http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy index 32a1975..ae0b252 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/groovy/org/apache/nifi/properties/ConfigEncryptionToolTest.groovy @@ -19,6 +19,7 @@ package org.apache.nifi.properties import org.apache.commons.lang3.SystemUtils import org.apache.log4j.AppenderSkeleton import org.apache.log4j.spi.LoggingEvent +import org.apache.nifi.encrypt.StringEncryptor import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException import org.apache.nifi.util.NiFiProperties import org.apache.nifi.util.console.TextDevice @@ -41,6 +42,10 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import javax.crypto.Cipher +import javax.crypto.SecretKey +import javax.crypto.SecretKeyFactory +import javax.crypto.spec.PBEKeySpec +import javax.crypto.spec.PBEParameterSpec import java.nio.file.Files import java.nio.file.attribute.PosixFilePermission import java.security.KeyException @@ -68,9 +73,20 @@ class ConfigEncryptionToolTest extends GroovyTestCase { private static final String PASSWORD_KEY_HEX = isUnlimitedStrengthCryptoAvailable() ? PASSWORD_KEY_HEX_256 : PASSWORD_KEY_HEX_128 + // Known issue documented in NIFI-1465 and NIFI-1255 where the password cannot be > 16 characters without the JCE unlimited strength policies installed + private static final String FLOW_PASSWORD_128 = "shortPassword" + private static final String FLOW_PASSWORD_256 = "thisIsABadPassword" + public static + final String FLOW_PASSWORD = isUnlimitedStrengthCryptoAvailable() ? FLOW_PASSWORD_256 : FLOW_PASSWORD_128 + private static final int LIP_PASSWORD_LINE_COUNT = 3 private final String PASSWORD_PROP_REGEX = "<property[^>]* name=\".* Password\"" + private static final String DEFAULT_ALGORITHM = "PBEWITHMD5AND256BITAES-CBC-OPENSSL" + private static final String DEFAULT_PROVIDER = "BC" + private static final String WFXCTR = ConfigEncryptionTool.WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX + private final String DEFAULT_LEGACY_SENSITIVE_PROPS_KEY = "nififtw!" + @BeforeClass public static void setUpOnce() throws Exception { Security.addProvider(new BouncyCastleProvider()) @@ -1269,7 +1285,14 @@ class ConfigEncryptionToolTest extends GroovyTestCase { assert updatedLines.contains("${key}=${niFiProperties.getProperty(key)}".toString()) } - assert originalLines == updatedLines + if (originalLines.size() != updatedLines.size()) { + // In situations where the original nifi.properties did not have a protection scheme for nifi.sensitive.props.key, it is added automatically now + def differentLines = updatedLines - originalLines + assert differentLines.size() == 1 + assert differentLines.first() == "nifi.sensitive.props.key.protected=" + } else { + assert originalLines == updatedLines + } logger.info("Updated nifi.properties:") logger.info("\n" * 2 + updatedLines.join("\n")) @@ -1304,7 +1327,14 @@ class ConfigEncryptionToolTest extends GroovyTestCase { assert updatedLines.contains("${key}=${niFiProperties.getProperty(key)}".toString()) } - assert originalLines == updatedLines + if (originalLines.size() != updatedLines.size()) { + // In situations where the original nifi.properties did not have a protection scheme for nifi.sensitive.props.key, it is added automatically now + def differentLines = updatedLines - originalLines + assert differentLines.size() == 1 + assert differentLines.first() == "nifi.sensitive.props.key.protected=" + } else { + assert originalLines == updatedLines + } logger.info("Updated nifi.properties:") logger.info("\n" * 2 + updatedLines.join("\n")) @@ -2308,7 +2338,9 @@ class ConfigEncryptionToolTest extends GroovyTestCase { // Assert assert serializedLines == encryptedLines - assert TestAppender.events.any { it.renderedMessage =~ "No provider element with class org.apache.nifi.ldap.LdapProvider found in XML content; the file could be empty or the element may be missing or commented out" } + assert TestAppender.events.any { + it.renderedMessage =~ "No provider element with class org.apache.nifi.ldap.LdapProvider found in XML content; the file could be empty or the element may be missing or commented out" + } } @Test @@ -2337,7 +2369,9 @@ class ConfigEncryptionToolTest extends GroovyTestCase { // Assert assert serializedLines.findAll { it }.isEmpty() - assert TestAppender.events.any { it.renderedMessage =~ "No provider element with class org.apache.nifi.ldap.LdapProvider found in XML content; the file could be empty or the element may be missing or commented out" } + assert TestAppender.events.any { + it.renderedMessage =~ "No provider element with class org.apache.nifi.ldap.LdapProvider found in XML content; the file could be empty or the element may be missing or commented out" + } } @Test @@ -2616,6 +2650,945 @@ class ConfigEncryptionToolTest extends GroovyTestCase { // Assertions defined above } + + @Test + void testParseShouldIgnoreFilesIfOverrideFlagPresent() { + // Arrange + String niFiPropertiesPath = "conf/nifi.properties" + String flowXmlPath = "conf/flow.xml.gz" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + tool.parse("-n ${niFiPropertiesPath} -f ${flowXmlPath} -x".split(" ") as String[]) + + // Assert + assert !tool.handlingNiFiProperties + assert !tool.handlingLoginIdentityProviders + assert tool.handlingFlowXml + } + + @Test + void testParseShouldWarnIfFlowXmlWillBeOverwritten() { + // Arrange + String niFiPropertiesPath = "conf/nifi.properties" + String flowXmlPath = "conf/flow.xml.gz" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + tool.parse("-n ${niFiPropertiesPath} -f ${flowXmlPath}".split(" ") as String[]) + logger.info("Parsed nifi.properties location: ${tool.niFiPropertiesPath}") + logger.info("Parsed flow.xml.gz location: ${tool.flowXmlPath}") + logger.info("Parsed output flow.xml.gz location: ${tool.outputFlowXmlPath}") + + // Assert + assert !TestAppender.events.isEmpty() + assert TestAppender.events.any { + it.message =~ "The source flow.xml.gz and destination flow.xml.gz are identical \\[.*\\] so the original will be overwritten" + } + } + + @Test + void testParseShouldFailOnFlowWithoutNiFiProperties() { + // Arrange + String flowXmlPath = "conf/flow.xml.gz" + ConfigEncryptionTool tool = new ConfigEncryptionTool() + + // Act + def msg = shouldFail(CommandLineParseException) { + tool.parse("-f ${flowXmlPath} -x".split(" ") as String[]) + } + logger.expected(msg) + + // Assert + assert msg == "In order to migrate a flow.xml.gz, a nifi.properties file must also be specified via '-n'/'--niFiProperties'." as String + } + + // TODO: Test different algs/providers + // TODO: Test reading sensitive props key from console + // TODO: All combo scenarios + @Test + void testShouldPerformFullOperationOnFlowXmlWithoutEncryptedNiFiProperties() { + // Arrange + exit.expectSystemExitWithStatus(0) + + File tmpDir = setupTmpDir() + + File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_master_key.conf") + File bootstrapFile = new File("target/tmp/tmp_bootstrap.conf") + bootstrapFile.delete() + + Files.copy(emptyKeyFile.toPath(), bootstrapFile.toPath()) + final List<String> originalBootstrapLines = bootstrapFile.readLines() + String originalKeyLine = originalBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Original key line from bootstrap.conf: ${originalKeyLine}") + assert originalKeyLine == ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + + // Not "handling" NFP, so update in place (not source test resource) + String niFiPropertiesTemplatePath = "src/test/resources/nifi_default.properties" + File niFiPropertiesFile = new File(niFiPropertiesTemplatePath) + + File workingNiFiPropertiesFile = new File("target/tmp/tmp-nifi.properties") + workingNiFiPropertiesFile.delete() + Files.copy(niFiPropertiesFile.toPath(), workingNiFiPropertiesFile.toPath()) + + File flowXmlFile = new File("src/test/resources/flow.xml.gz") + File workingFlowXmlFile = new File("target/tmp/tmp-flow.xml.gz") + workingFlowXmlFile.delete() + Files.copy(flowXmlFile.toPath(), workingFlowXmlFile.toPath()) + + // Read the uncompressed version to compare later + File originalFlowXmlFile = new File("src/test/resources/flow.xml") + final String ORIGINAL_FLOW_XML_CONTENT = originalFlowXmlFile.text + def originalFlowCipherTexts = ORIGINAL_FLOW_XML_CONTENT.findAll(WFXCTR) + final int CIPHER_TEXT_COUNT = originalFlowCipherTexts.size() + + NiFiProperties inputProperties = new NiFiPropertiesLoader().load(workingNiFiPropertiesFile) + logger.info("Loaded ${inputProperties.size()} properties from input file") + ProtectedNiFiProperties protectedInputProperties = new ProtectedNiFiProperties(inputProperties) + def originalSensitiveValues = protectedInputProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): protectedInputProperties.getProperty(key)] } + logger.info("Original sensitive values: ${originalSensitiveValues}") + + String newFlowPassword = FLOW_PASSWORD + + String[] args = ["-n", workingNiFiPropertiesFile.path, "-f", workingFlowXmlFile.path, "-x", "-v", "-s", newFlowPassword] + + exit.checkAssertionAfterwards(new Assertion() { + public void checkAssertion() { + final List<String> updatedPropertiesLines = workingNiFiPropertiesFile.readLines() + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + updatedPropertiesLines.join("\n")) + + // Check that the output values for everything is the same except the sensitive props key + NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(workingNiFiPropertiesFile) + assert updatedProperties.size() == inputProperties.size() + assert updatedProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY) == newFlowPassword + originalSensitiveValues.every { String key, String originalValue -> + if (key != NiFiProperties.SENSITIVE_PROPS_KEY) { + assert updatedProperties.getProperty(key) == originalValue + } + } + + // Check that bootstrap.conf did not change + final List<String> updatedBootstrapLines = bootstrapFile.readLines() + String updatedKeyLine = updatedBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Updated key line: ${updatedKeyLine}") + + assert updatedKeyLine == EXPECTED_KEY_LINE + assert originalBootstrapLines.size() == updatedBootstrapLines.size() + + // Verify the flow definition + def verifyTool = new ConfigEncryptionTool() + verifyTool.isVerbose = true + verifyTool.flowXmlPath = workingFlowXmlFile.path + String updatedFlowXmlContent = verifyTool.loadFlowXml() + + // Check that the flow.xml.gz content changed + assert updatedFlowXmlContent != ORIGINAL_FLOW_XML_CONTENT + + // Verify that the cipher texts decrypt correctly + logger.info("Original flow.xml.gz cipher texts: ${originalFlowCipherTexts}") + def flowCipherTexts = updatedFlowXmlContent.findAll(WFXCTR) + logger.info("Updated flow.xml.gz cipher texts: ${flowCipherTexts}") + assert flowCipherTexts.size() == CIPHER_TEXT_COUNT + flowCipherTexts.every { + assert ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) == "thisIsABadPassword" + } + } + }); + + // Act + ConfigEncryptionTool.main(args) + logger.info("Invoked #main with ${args.join(" ")}") + + // Assert + + // Assertions defined above + } + + /** + * In this scenario, the nifi.properties is not encrypted and the flow.xml.gz is "migrated" from Key X to the same key (the default key). + */ + @Test + void testShouldPerformFullOperationOnFlowXmlWithSameSensitivePropsKey() { + // Arrange + exit.expectSystemExitWithStatus(0) + + File tmpDir = setupTmpDir() + + File emptyKeyFile = new File("src/test/resources/bootstrap_with_empty_master_key.conf") + File bootstrapFile = new File("target/tmp/tmp_bootstrap.conf") + bootstrapFile.delete() + + Files.copy(emptyKeyFile.toPath(), bootstrapFile.toPath()) + final List<String> originalBootstrapLines = bootstrapFile.readLines() + String originalKeyLine = originalBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Original key line from bootstrap.conf: ${originalKeyLine}") + assert originalKeyLine == ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + + // Not "handling" NFP, so update in place (not source test resource) + String niFiPropertiesTemplatePath = "src/test/resources/nifi_default.properties" + File niFiPropertiesFile = new File(niFiPropertiesTemplatePath) + + File workingNiFiPropertiesFile = new File("target/tmp/tmp-nifi.properties") + workingNiFiPropertiesFile.delete() + Files.copy(niFiPropertiesFile.toPath(), workingNiFiPropertiesFile.toPath()) + + File flowXmlFile = new File("src/test/resources/flow_default_key.xml.gz") + File workingFlowXmlFile = new File("target/tmp/tmp-flow.xml.gz") + workingFlowXmlFile.delete() + Files.copy(flowXmlFile.toPath(), workingFlowXmlFile.toPath()) + + // Read the uncompressed version to compare later + File originalFlowXmlFile = new File("src/test/resources/flow_default_key.xml") + final String ORIGINAL_FLOW_XML_CONTENT = originalFlowXmlFile.text + def originalFlowCipherTexts = ORIGINAL_FLOW_XML_CONTENT.findAll(WFXCTR) + final int CIPHER_TEXT_COUNT = originalFlowCipherTexts.size() + + NiFiProperties inputProperties = new NiFiPropertiesLoader().load(workingNiFiPropertiesFile) + logger.info("Loaded ${inputProperties.size()} properties from input file") + ProtectedNiFiProperties protectedInputProperties = new ProtectedNiFiProperties(inputProperties) + def originalSensitiveValues = protectedInputProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): protectedInputProperties.getProperty(key)] } + logger.info("Original sensitive values: ${originalSensitiveValues}") + + String newFlowPassword = DEFAULT_LEGACY_SENSITIVE_PROPS_KEY + + String[] args = ["-n", workingNiFiPropertiesFile.path, "-f", workingFlowXmlFile.path, "-x", "-v", "-s", newFlowPassword] + + exit.checkAssertionAfterwards(new Assertion() { + public void checkAssertion() { + final List<String> updatedPropertiesLines = workingNiFiPropertiesFile.readLines() + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + updatedPropertiesLines.join("\n")) + + // Check that the output values for everything is the same including the sensitive props key + NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(workingNiFiPropertiesFile) + assert updatedProperties.size() == inputProperties.size() + originalSensitiveValues.every { String key, String originalValue -> + assert updatedProperties.getProperty(key) == originalValue + } + + // Check that bootstrap.conf did not change + final List<String> updatedBootstrapLines = bootstrapFile.readLines() + String updatedKeyLine = updatedBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Updated key line: ${updatedKeyLine}") + + assert updatedKeyLine == EXPECTED_KEY_LINE + assert originalBootstrapLines.size() == updatedBootstrapLines.size() + + // Verify the flow definition + def verifyTool = new ConfigEncryptionTool() + verifyTool.isVerbose = true + verifyTool.flowXmlPath = workingFlowXmlFile.path + String updatedFlowXmlContent = verifyTool.loadFlowXml() + + // Check that the flow.xml.gz cipher texts did change (new salt) + assert updatedFlowXmlContent != ORIGINAL_FLOW_XML_CONTENT + + // Verify that the cipher texts decrypt correctly + logger.info("Original flow.xml.gz cipher texts: ${originalFlowCipherTexts}") + def flowCipherTexts = updatedFlowXmlContent.findAll(WFXCTR) + logger.info("Updated flow.xml.gz cipher texts: ${flowCipherTexts}") + assert flowCipherTexts.size() == CIPHER_TEXT_COUNT + flowCipherTexts.every { + assert ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) == "thisIsABadPassword" + } + } + }); + + // Act + ConfigEncryptionTool.main(args) + logger.info("Invoked #main with ${args.join(" ")}") + + // Assert + + // Assertions defined above + } + + /** + * In this scenario, the nifi.properties file has a sensitive key value which is already encrypted. The goal is to provide a new provide a new sensitive key value, perform the migration of the flow.xml.gz, and update nifi.properties with a new encrypted sensitive key value without modifying any other nifi.properties values. + */ + @Test + void testShouldPerformFullOperationOnFlowXmlWithPreviouslyEncryptedNiFiProperties() { + // Arrange + exit.expectSystemExitWithStatus(0) + + File tmpDir = setupTmpDir() + + File passwordKeyFile = new File("src/test/resources/bootstrap_with_master_key_password_128.conf") + File bootstrapFile = new File("target/tmp/tmp_bootstrap.conf") + bootstrapFile.delete() + + Files.copy(passwordKeyFile.toPath(), bootstrapFile.toPath()) + final List<String> originalBootstrapLines = bootstrapFile.readLines() + String originalKeyLine = originalBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + PASSWORD_KEY_HEX_128 + logger.info("Original key line from bootstrap.conf: ${originalKeyLine}") + assert originalKeyLine == ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + PASSWORD_KEY_HEX_128 + + // Not "handling" NFP, so update in place (not source test resource) + String niFiPropertiesTemplatePath = "src/test/resources/nifi_with_few_sensitive_properties_protected_aes_password_128.properties" + File niFiPropertiesFile = new File(niFiPropertiesTemplatePath) + + File workingNiFiPropertiesFile = new File("target/tmp/tmp-nifi.properties") + workingNiFiPropertiesFile.delete() + Files.copy(niFiPropertiesFile.toPath(), workingNiFiPropertiesFile.toPath()) + + // Use a flow definition that was encrypted with the hard-coded default SP key + File flowXmlFile = new File("src/test/resources/flow_default_key.xml.gz") + File workingFlowXmlFile = new File("target/tmp/tmp-flow.xml.gz") + workingFlowXmlFile.delete() + Files.copy(flowXmlFile.toPath(), workingFlowXmlFile.toPath()) + + // Read the uncompressed version to compare later + File originalFlowXmlFile = new File("src/test/resources/flow_default_key.xml") + final String ORIGINAL_FLOW_XML_CONTENT = originalFlowXmlFile.text + def originalFlowCipherTexts = ORIGINAL_FLOW_XML_CONTENT.findAll(WFXCTR) + final int CIPHER_TEXT_COUNT = originalFlowCipherTexts.size() + + // Load both the encrypted and decrypted properties to compare later + NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(PASSWORD_KEY_HEX_128) + NiFiProperties inputProperties = niFiPropertiesLoader.load(workingNiFiPropertiesFile) + logger.info("Loaded ${inputProperties.size()} properties from input file") + ProtectedNiFiProperties protectedInputProperties = new ProtectedNiFiProperties(inputProperties) + def originalSensitiveValues = protectedInputProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): protectedInputProperties.getProperty(key)] } + logger.info("Original sensitive values: ${originalSensitiveValues}") + + + final String SENSITIVE_PROTECTION_KEY = ProtectedNiFiProperties.getProtectionKey(NiFiProperties.SENSITIVE_PROPS_KEY) + ProtectedNiFiProperties encryptedProperties = niFiPropertiesLoader.readProtectedPropertiesFromDisk(workingNiFiPropertiesFile) + def originalEncryptedValues = encryptedProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): encryptedProperties.getProperty(key)] } + logger.info("Original encrypted values: ${originalEncryptedValues}") + String originalSensitiveKeyProtectionScheme = encryptedProperties.getProperty(SENSITIVE_PROTECTION_KEY) + logger.info("Sensitive property key originally protected with ${originalSensitiveKeyProtectionScheme}") + + String newFlowPassword = FLOW_PASSWORD + + // Bootstrap path must be provided to decrypt nifi.properties to get SP key + String[] args = ["-n", workingNiFiPropertiesFile.path, "-f", workingFlowXmlFile.path, "-b", bootstrapFile.path, "-x", "-v", "-s", newFlowPassword] + + exit.checkAssertionAfterwards(new Assertion() { + public void checkAssertion() { + final List<String> updatedPropertiesLines = workingNiFiPropertiesFile.readLines() + logger.info("Updated nifi.properties:") + logger.info("\n" * 2 + updatedPropertiesLines.join("\n")) + + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(PASSWORD_KEY_HEX_128) + + // Check that the output values for everything is the same except the sensitive props key + NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(workingNiFiPropertiesFile) + assert updatedProperties.size() == inputProperties.size() + String newSensitivePropertyKey = updatedProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY) + + // Check that the encrypted value changed + assert newSensitivePropertyKey != originalSensitiveValues.get(NiFiProperties.SENSITIVE_PROPS_KEY) + + // Check that the decrypted value is the new password + assert spp.unprotect(newSensitivePropertyKey) == newFlowPassword + + // Check that all other values stayed the same + originalEncryptedValues.every { String key, String originalValue -> + if (key != NiFiProperties.SENSITIVE_PROPS_KEY) { + assert updatedProperties.getProperty(key) == originalValue + } + } + + // Check that all other (decrypted) values stayed the same + originalSensitiveValues.every { String key, String originalValue -> + if (key != NiFiProperties.SENSITIVE_PROPS_KEY) { + assert spp.unprotect(updatedProperties.getProperty(key)) == originalValue + } + } + + // Check that the protection scheme did not change + String sensitiveKeyProtectionScheme = updatedProperties.getProperty(SENSITIVE_PROTECTION_KEY) + logger.info("Sensitive property key currently protected with ${sensitiveKeyProtectionScheme}") + assert sensitiveKeyProtectionScheme == originalSensitiveKeyProtectionScheme + + // Check that bootstrap.conf did not change + final List<String> updatedBootstrapLines = bootstrapFile.readLines() + String updatedKeyLine = updatedBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Updated key line: ${updatedKeyLine}") + + assert updatedKeyLine == EXPECTED_KEY_LINE + assert originalBootstrapLines.size() == updatedBootstrapLines.size() + + // Verify the flow definition + def verifyTool = new ConfigEncryptionTool() + verifyTool.isVerbose = true + verifyTool.flowXmlPath = workingFlowXmlFile.path + String updatedFlowXmlContent = verifyTool.loadFlowXml() + + // Check that the flow.xml.gz content changed + assert updatedFlowXmlContent != ORIGINAL_FLOW_XML_CONTENT + + // Verify that the cipher texts decrypt correctly + logger.info("Original flow.xml.gz cipher texts: ${originalFlowCipherTexts}") + def flowCipherTexts = updatedFlowXmlContent.findAll(WFXCTR) + logger.info("Updated flow.xml.gz cipher texts: ${flowCipherTexts}") + assert flowCipherTexts.size() == CIPHER_TEXT_COUNT + flowCipherTexts.every { + assert ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) == "thisIsABadPassword" + } + } + }); + + // Act + ConfigEncryptionTool.main(args) + logger.info("Invoked #main with ${args.join(" ")}") + + // Assert + + // Assertions defined above + } + + /** + * In this scenario, the nifi.properties file has a sensitive key value which is already encrypted. The goal is to provide a new provide a new sensitive key value, perform the migration of the flow.xml.gz, and update nifi.properties with a new encrypted sensitive key value without modifying any other nifi.properties values, and repeat this process multiple times to ensure no corruption of the keys. + */ + @Test + void testShouldPerformFullOperationOnFlowXmlMultipleTimes() { + // Arrange + exit.expectSystemExitWithStatus(0) + + File tmpDir = setupTmpDir() + + File passwordKeyFile = new File("src/test/resources/bootstrap_with_master_key_password_128.conf") + File bootstrapFile = new File("target/tmp/tmp_bootstrap.conf") + bootstrapFile.delete() + + Files.copy(passwordKeyFile.toPath(), bootstrapFile.toPath()) + final List<String> originalBootstrapLines = bootstrapFile.readLines() + String originalKeyLine = originalBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + final String EXPECTED_KEY_LINE = ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + PASSWORD_KEY_HEX_128 + logger.info("Original key line from bootstrap.conf: ${originalKeyLine}") + assert originalKeyLine == ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX + PASSWORD_KEY_HEX_128 + + // Not "handling" NFP, so update in place (not source test resource) + String niFiPropertiesTemplatePath = "src/test/resources/nifi_with_few_sensitive_properties_protected_aes_password_128.properties" + File niFiPropertiesFile = new File(niFiPropertiesTemplatePath) + + File workingNiFiPropertiesFile = new File("target/tmp/tmp-nifi.properties") + workingNiFiPropertiesFile.delete() + Files.copy(niFiPropertiesFile.toPath(), workingNiFiPropertiesFile.toPath()) + + // Use a flow definition that was encrypted with the hard-coded default SP key + File flowXmlFile = new File("src/test/resources/flow_default_key.xml.gz") + File workingFlowXmlFile = new File("target/tmp/tmp-flow.xml.gz") + workingFlowXmlFile.delete() + Files.copy(flowXmlFile.toPath(), workingFlowXmlFile.toPath()) + + // Read the uncompressed version to compare later + File originalFlowXmlFile = new File("src/test/resources/flow_default_key.xml") + final String ORIGINAL_FLOW_XML_CONTENT = originalFlowXmlFile.text + def originalFlowCipherTexts = ORIGINAL_FLOW_XML_CONTENT.findAll(WFXCTR) + final int CIPHER_TEXT_COUNT = originalFlowCipherTexts.size() + + // Load both the encrypted and decrypted properties to compare later + NiFiPropertiesLoader niFiPropertiesLoader = NiFiPropertiesLoader.withKey(PASSWORD_KEY_HEX_128) + NiFiProperties inputProperties = niFiPropertiesLoader.load(workingNiFiPropertiesFile) + logger.info("Loaded ${inputProperties.size()} properties from input file") + ProtectedNiFiProperties protectedInputProperties = new ProtectedNiFiProperties(inputProperties) + def originalSensitiveValues = protectedInputProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): protectedInputProperties.getProperty(key)] } + logger.info("Original sensitive values: ${originalSensitiveValues}") + + final String SENSITIVE_PROTECTION_KEY = ProtectedNiFiProperties.getProtectionKey(NiFiProperties.SENSITIVE_PROPS_KEY) + ProtectedNiFiProperties encryptedProperties = niFiPropertiesLoader.readProtectedPropertiesFromDisk(workingNiFiPropertiesFile) + def originalEncryptedValues = encryptedProperties.getSensitivePropertyKeys().collectEntries { String key -> [(key): encryptedProperties.getProperty(key)] } + logger.info("Original encrypted values: ${originalEncryptedValues}") + String originalSensitiveKeyProtectionScheme = encryptedProperties.getProperty(SENSITIVE_PROTECTION_KEY) + logger.info("Sensitive property key originally protected with ${originalSensitiveKeyProtectionScheme}") + + // Create a series of passwords with which to encrypt the flow XML, starting with the current password + def passwordProgression = [DEFAULT_LEGACY_SENSITIVE_PROPS_KEY] + (0..5).collect { "${FLOW_PASSWORD}${it}" } + + // The master key is not changing + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(PASSWORD_KEY_HEX_128) + + // Act + passwordProgression.eachWithIndex { String existingFlowPassword, int i -> + if (i < passwordProgression.size() - 1) { + exit.expectSystemExitWithStatus(0) + String newFlowPassword = passwordProgression[i + 1] + logger.info("Migrating from ${existingFlowPassword} to ${newFlowPassword}") + + // Set up assertions for this iteration +// exit.expectSystemExitWithStatus(0) +// exit.checkAssertionAfterwards(new Assertion() { +// public void checkAssertion() { + +// } +// }); + + // Bootstrap path must be provided to decrypt nifi.properties to get SP key + String[] args = ["-n", workingNiFiPropertiesFile.path, "-f", workingFlowXmlFile.path, "-b", bootstrapFile.path, "-x", "-v", "-s", newFlowPassword] + + def msg = shouldFail { + logger.info("Invoked #main with ${args.join(" ")}") + ConfigEncryptionTool.main(args) + } + logger.expected(msg) + + // Assert + // Get the updated nifi.properties and check the sensitive key + final List<String> updatedPropertiesLines = workingNiFiPropertiesFile.readLines() +// logger.info("Updated nifi.properties:") +// logger.info("\n" * 2 + updatedPropertiesLines.join("\n")) + String updatedSensitiveKeyLine = updatedPropertiesLines.find { it.startsWith(NiFiProperties.SENSITIVE_PROPS_KEY) } + logger.info("Updated key line: ${updatedSensitiveKeyLine}") + + // Check that the output values for everything are the same except the sensitive props key + NiFiProperties updatedProperties = new NiFiPropertiesLoader().readProtectedPropertiesFromDisk(workingNiFiPropertiesFile) + assert updatedProperties.size() == inputProperties.size() + String newSensitivePropertyKey = updatedProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY) + + // Check that the encrypted value changed + assert newSensitivePropertyKey != originalSensitiveValues.get(NiFiProperties.SENSITIVE_PROPS_KEY) + + // Check that the decrypted value is the new password + assert spp.unprotect(newSensitivePropertyKey) == newFlowPassword + + // Check that all other values stayed the same + originalEncryptedValues.every { String key, String originalValue -> + if (key != NiFiProperties.SENSITIVE_PROPS_KEY) { + assert updatedProperties.getProperty(key) == originalValue + } + } + + // Check that all other (decrypted) values stayed the same + originalSensitiveValues.every { String key, String originalValue -> + if (key != NiFiProperties.SENSITIVE_PROPS_KEY) { + assert spp.unprotect(updatedProperties.getProperty(key)) == originalValue + } + } + + // Check that the protection scheme did not change + String sensitiveKeyProtectionScheme = updatedProperties.getProperty(SENSITIVE_PROTECTION_KEY) + logger.info("Sensitive property key currently protected with ${sensitiveKeyProtectionScheme}") + assert sensitiveKeyProtectionScheme == originalSensitiveKeyProtectionScheme + + // Check that bootstrap.conf did not change + final List<String> updatedBootstrapLines = bootstrapFile.readLines() + String updatedKeyLine = updatedBootstrapLines.find { + it.startsWith(ConfigEncryptionTool.BOOTSTRAP_KEY_PREFIX) + } + logger.info("Updated key line: ${updatedKeyLine}") + + assert updatedKeyLine == EXPECTED_KEY_LINE + assert originalBootstrapLines.size() == updatedBootstrapLines.size() + + // Verify the flow definition + def verifyTool = new ConfigEncryptionTool() + verifyTool.isVerbose = true + verifyTool.flowXmlPath = workingFlowXmlFile.path + String updatedFlowXmlContent = verifyTool.loadFlowXml() + + // Check that the flow.xml.gz content changed + assert updatedFlowXmlContent != ORIGINAL_FLOW_XML_CONTENT + + // Verify that the cipher texts decrypt correctly + logger.info("Original flow.xml.gz cipher texts: ${originalFlowCipherTexts}") + def flowCipherTexts = updatedFlowXmlContent.findAll(WFXCTR) + logger.info("Updated flow.xml.gz cipher texts: ${flowCipherTexts}") + assert flowCipherTexts.size() == CIPHER_TEXT_COUNT + flowCipherTexts.every { + assert ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) == "thisIsABadPassword" + } + + // Update the "original" flow cipher texts for the next run to the current values + originalFlowCipherTexts = flowCipherTexts + } + } + } + + @Test + void testDecryptFlowXmlContentShouldVerifyPattern() { + // Arrange + String existingFlowPassword = "flowPassword" + final String DEFAULT_ALGORITHM = "PBEWITHMD5AND256BITAES-CBC-OPENSSL" + final String DEFAULT_PROVIDER = "BC" + + String sensitivePropertyValue = "thisIsABadProcessorPassword" + + StringEncryptor sanityEncryptor = new StringEncryptor(DEFAULT_ALGORITHM, DEFAULT_PROVIDER, existingFlowPassword) + String sanityCipherText = "enc{${sanityEncryptor.encrypt(sensitivePropertyValue)}}" + logger.info("Sanity check value: \t${sensitivePropertyValue} -> ${sanityCipherText}") + + def validCipherTexts = (0..4).collect { + "enc{${sanityEncryptor.encrypt(sensitivePropertyValue)}}" + } + logger.info("Generated valid cipher texts: \n${validCipherTexts.join("\n")}") + + def invalidCipherTexts = ["enc{}", + "enc{x}", + "encx", + "enc{012}", + "enc{01", + "enc{aBc19+===}", + "enc{aB=c19+}", + "enc{aB@}", + "", + "}", + "\"", + ">", + null] + + // Act + def successfulResults = validCipherTexts.collect { String cipherText -> + ConfigEncryptionTool.decryptFlowElement(cipherText, existingFlowPassword) + } + + def failedResults = invalidCipherTexts.collect { String cipherText -> + def msg = shouldFail(SensitivePropertyProtectionException) { + ConfigEncryptionTool.decryptFlowElement(cipherText, existingFlowPassword) + } + logger.expected(msg) + msg + } + + // Assert + assert successfulResults.every { it == sensitivePropertyValue } + assert failedResults.every { + it =~ /The provided cipher text does not match the expected format 'enc\{0123456789ABCDEF\.\.\.\}'/ || + it == "The provided cipher text must have an even number of hex characters" + } + } + + /** + * This test verifies that the crypto logic in the tool is compatible with the default {@link StringEncryptor} implementation. + */ + @Test + void testShouldDecryptFlowXmlContent() { + // Arrange + String existingFlowPassword = "flowPassword" + final String DEFAULT_ALGORITHM = "PBEWITHMD5AND256BITAES-CBC-OPENSSL" + final String DEFAULT_PROVIDER = "BC" + + String sensitivePropertyValue = "thisIsABadProcessorPassword" + + StringEncryptor sanityEncryptor = new StringEncryptor(DEFAULT_ALGORITHM, DEFAULT_PROVIDER, existingFlowPassword) + String sanityCipherText = "enc{${sanityEncryptor.encrypt(sensitivePropertyValue)}}" + logger.info("Sanity check value: \t${sensitivePropertyValue} -> ${sanityCipherText}") + + // Act + String decryptedElement = ConfigEncryptionTool.decryptFlowElement(sanityCipherText, existingFlowPassword, DEFAULT_ALGORITHM, DEFAULT_PROVIDER) + logger.info("Decrypted flow element: ${decryptedElement}") + String decryptedElementWithDefaultParameters = ConfigEncryptionTool.decryptFlowElement(sanityCipherText, existingFlowPassword) + logger.info("Decrypted flow element: ${decryptedElementWithDefaultParameters}") + + // Assert + assert decryptedElement == sensitivePropertyValue + assert decryptedElementWithDefaultParameters == sensitivePropertyValue + } + + /** + * This test verifies that the crypto logic in the tool is compatible with an encrypted value taken from a production flow.xml.gz. + */ + @Test + void testShouldDecryptFlowXmlContentFromLegacyFlow() { + // Arrange + + // StringEncryptor.DEFAULT_SENSITIVE_PROPS_KEY = "nififtw!" at the time this test + // was written and for the encrypted value, but it could change, so don't + // reference transitively here + String existingFlowPassword = DEFAULT_LEGACY_SENSITIVE_PROPS_KEY + final String DEFAULT_ALGORITHM = "PBEWITHMD5AND256BITAES-CBC-OPENSSL" + final String DEFAULT_PROVIDER = "BC" + + final String EXPECTED_PLAINTEXT = "thisIsABadPassword" + + final String ENCRYPTED_VALUE_FROM_FLOW = "enc{2032416987A00D9FCD757528D7AE609D7E793CA5F956641DB53E14CDB9BFCD4037B73AC705CD3F5C1C1BDE18B8D7B281}" + + // Act + String decryptedElement = ConfigEncryptionTool.decryptFlowElement(ENCRYPTED_VALUE_FROM_FLOW, existingFlowPassword, DEFAULT_ALGORITHM, DEFAULT_PROVIDER) + logger.info("Decrypted flow element: ${decryptedElement}") + + // Assert + assert decryptedElement == EXPECTED_PLAINTEXT + } + + @Test + void testShouldEncryptFlowXmlContent() { + // Arrange + String flowPassword = "flowPassword" + String sensitivePropertyValue = "thisIsABadProcessorPassword" + byte[] saltBytes = "thisIsABadSalt..".bytes + + StringEncryptor sanityEncryptor = new StringEncryptor(DEFAULT_ALGORITHM, DEFAULT_PROVIDER, flowPassword) + + Cipher encryptionCipher = generateEncryptionCipher(flowPassword) + + // Act + String encryptedElement = ConfigEncryptionTool.encryptFlowElement(sensitivePropertyValue, saltBytes, encryptionCipher) + logger.info("Encrypted flow element: ${encryptedElement}") + + // Assert + assert encryptedElement =~ WFXCTR + String sanityPlaintext = sanityEncryptor.decrypt(encryptedElement[4..<-1]) + logger.info("Sanity check value: \t${encryptedElement} -> ${sanityPlaintext}") + + assert sanityPlaintext == sensitivePropertyValue + } + + @Test + void testShouldEncryptAndDecryptFlowXmlContent() { + // Arrange + String flowPassword = "flowPassword" + String sensitivePropertyValue = "thisIsABadProcessorPassword" + byte[] saltBytes = "thisIsABadSalt..".bytes + + Cipher encryptionCipher = generateEncryptionCipher(flowPassword) + + // Act + String encryptedElement = ConfigEncryptionTool.encryptFlowElement(sensitivePropertyValue, saltBytes, encryptionCipher) + logger.info("Encrypted flow element: ${encryptedElement}") + + String decryptedElement = ConfigEncryptionTool.decryptFlowElement(encryptedElement, flowPassword) + logger.info("Decrypted flow element: ${decryptedElement}") + + // Assert + assert encryptedElement =~ WFXCTR + assert decryptedElement == sensitivePropertyValue + } + + private + static Cipher generateEncryptionCipher(String password, String algorithm = DEFAULT_ALGORITHM, String provider = DEFAULT_PROVIDER) { + Cipher cipher = Cipher.getInstance(algorithm, provider) + PBEKeySpec keySpec = new PBEKeySpec(password.chars) + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(algorithm, provider) + SecretKey pbeKey = keyFactory.generateSecret(keySpec) + byte[] saltBytes = "thisIsABadSalt..".bytes + PBEParameterSpec parameterSpec = new PBEParameterSpec(saltBytes, 1000) + cipher.init(Cipher.ENCRYPT_MODE, pbeKey, parameterSpec) + cipher + } + + @Test + void testShouldMigrateFlowXmlContent() { + // Arrange + String flowXmlPath = "src/test/resources/flow.xml" + File flowXmlFile = new File(flowXmlPath) + + File tmpDir = setupTmpDir() + + File workingFile = new File("target/tmp/tmp-flow.xml") + workingFile.delete() + Files.copy(flowXmlFile.toPath(), workingFile.toPath()) + ConfigEncryptionTool tool = new ConfigEncryptionTool() + tool.isVerbose = true + + final String SENSITIVE_VALUE = "thisIsABadPassword" + + String existingFlowPassword = DEFAULT_LEGACY_SENSITIVE_PROPS_KEY + String newFlowPassword = FLOW_PASSWORD + + String xmlContent = workingFile.text + logger.info("Read flow.xml: \n${xmlContent}") + + // There are two encrypted passwords in this flow + int cipherTextCount = xmlContent.findAll(WFXCTR).size() + logger.info("Found ${cipherTextCount} encrypted properties in the original flow.xml content") + + // Act + String migratedXmlContent = tool.migrateFlowXmlContent(xmlContent, existingFlowPassword, newFlowPassword) + logger.info("Migrated flow.xml: \n${migratedXmlContent}") + + // Assert + def newCipherTexts = migratedXmlContent.findAll(WFXCTR) + + assert newCipherTexts.size() == cipherTextCount + newCipherTexts.every { + assert ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) == SENSITIVE_VALUE + } + + // Ensure that everything else is identical + assert migratedXmlContent.replaceAll(WFXCTR, "") == + xmlContent.replaceAll(WFXCTR, "") + } + + + @Test + void testShouldMigrateFlowXmlContentMultipleTimes() { + // Arrange + String flowXmlPath = "src/test/resources/flow.xml" + File flowXmlFile = new File(flowXmlPath) + + File tmpDir = setupTmpDir() + + File workingFile = new File("target/tmp/tmp-flow.xml") + workingFile.delete() + Files.copy(flowXmlFile.toPath(), workingFile.toPath()) + ConfigEncryptionTool tool = new ConfigEncryptionTool() + tool.isVerbose = true + + final String SENSITIVE_VALUE = "thisIsABadPassword" + + // Create a series of passwords with which to encrypt the flow XML, starting with the current password + def passwordProgression = [DEFAULT_LEGACY_SENSITIVE_PROPS_KEY] + (0..5).collect { "${FLOW_PASSWORD}${it}" } + + String xmlContent = workingFile.text +// logger.info("Read flow.xml: \n${xmlContent}") + + // There are two encrypted passwords in this flow + final def ORIGINAL_CIPHER_TEXTS = xmlContent.findAll(WFXCTR) + logger.info("Cipher texts: \n${ORIGINAL_CIPHER_TEXTS.join("\n")}") + final int ORIGINAL_CIPHER_TEXT_COUNT = ORIGINAL_CIPHER_TEXTS.size() + logger.info("Found ${ORIGINAL_CIPHER_TEXT_COUNT} encrypted properties in the original flow.xml content") + + String currentXmlContent = xmlContent + + // Act + passwordProgression.eachWithIndex { String existingFlowPassword, int i -> + if (i < passwordProgression.size() - 1) { + String newFlowPassword = passwordProgression[i + 1] + logger.info("Migrating from ${existingFlowPassword} to ${newFlowPassword}") + + String migratedXmlContent = tool.migrateFlowXmlContent(currentXmlContent, existingFlowPassword, newFlowPassword) +// logger.info("Migrated flow.xml: \n${migratedXmlContent}") + + // Assert + def newCipherTexts = migratedXmlContent.findAll(WFXCTR) + logger.info("Cipher texts for iteration ${i}: \n${newCipherTexts.join("\n")}") + + assert newCipherTexts.size() == ORIGINAL_CIPHER_TEXT_COUNT + newCipherTexts.every { + assert ConfigEncryptionTool.decryptFlowElement(it, newFlowPassword) == SENSITIVE_VALUE + } + + // Ensure that everything else is identical + assert migratedXmlContent.replaceAll(WFXCTR, "") == + xmlContent.replaceAll(WFXCTR, "") + + // Update the "source" XML content for the next iteration + currentXmlContent = migratedXmlContent + } + } + } + + @Test + void testMigrateFlowXmlContentShouldUseConstantSalt() { + // Arrange + String flowXmlPath = "src/test/resources/flow.xml" + File flowXmlFile = new File(flowXmlPath) + + File tmpDir = setupTmpDir() + + File workingFile = new File("target/tmp/tmp-flow.xml") + workingFile.delete() + Files.copy(flowXmlFile.toPath(), workingFile.toPath()) + ConfigEncryptionTool tool = new ConfigEncryptionTool() + tool.isVerbose = true + + String existingFlowPassword = DEFAULT_LEGACY_SENSITIVE_PROPS_KEY + String newFlowPassword = FLOW_PASSWORD + + String xmlContent = workingFile.text + logger.info("Read flow.xml: \n${xmlContent}") + + // There are two encrypted passwords in this flow + int cipherTextCount = xmlContent.findAll(WFXCTR).size() + logger.info("Found ${cipherTextCount} encrypted properties in the original flow.xml content") + + // Act + String migratedXmlContent = tool.migrateFlowXmlContent(xmlContent, existingFlowPassword, newFlowPassword) + logger.info("Migrated flow.xml: \n${migratedXmlContent}") + + // Assert + def newCipherTexts = migratedXmlContent.findAll(WFXCTR) + + assert newCipherTexts.size() == cipherTextCount + + // Check that the same salt was used on all output + String saltHex = newCipherTexts.first()[4..<36] + logger.info("First detected salt: ${saltHex}") + newCipherTexts.every { + assert it[4..<36] == saltHex + } + } + + @Test + void testShouldLoadFlowXmlContent() { + // Arrange + String flowXmlPath = "src/test/resources/flow.xml" + File flowXmlFile = new File(flowXmlPath) + + String flowXmlGzPath = "src/test/resources/flow.xml.gz" + File flowXmlGzFile = new File(flowXmlGzPath) + + File tmpDir = setupTmpDir() + + File workingFile = new File("target/tmp/tmp-flow.xml") + workingFile.delete() + Files.copy(flowXmlFile.toPath(), workingFile.toPath()) + File workingGzFile = new File("target/tmp/tmp-flow.xml.gz") + workingGzFile.delete() + Files.copy(flowXmlGzFile.toPath(), workingGzFile.toPath()) + ConfigEncryptionTool tool = new ConfigEncryptionTool() + tool.isVerbose = true + tool.flowXmlPath = workingGzFile.path + + String xmlContent = workingFile.text + logger.info("Read flow.xml: \n${xmlContent}") + + // Act + String readXmlContent = tool.loadFlowXml() + logger.info("Loaded flow.xml.gz: \n${readXmlContent}") + + // Assert + assert readXmlContent == xmlContent + } + + @Test + void testShouldWriteFlowXmlToFile() { + // Arrange + String flowXmlPath = "src/test/resources/flow.xml" + File flowXmlFile = new File(flowXmlPath) + + String flowXmlGzPath = "src/test/resources/flow.xml.gz" + File flowXmlGzFile = new File(flowXmlGzPath) + + File tmpDir = setupTmpDir() + + File workingFile = new File("target/tmp/tmp-flow.xml") + workingFile.delete() + Files.copy(flowXmlFile.toPath(), workingFile.toPath()) + File workingGzFile = new File("target/tmp/tmp-flow.xml.gz") + workingGzFile.delete() + Files.copy(flowXmlGzFile.toPath(), workingGzFile.toPath()) + ConfigEncryptionTool tool = new ConfigEncryptionTool() + tool.isVerbose = true + tool.outputFlowXmlPath = workingGzFile.path.replaceAll("flow.xml.gz", "output.xml.gz") + + String xmlContent = workingFile.text + logger.info("Read flow.xml: \n${xmlContent}") + + // Act + tool.writeFlowXmlToFile(xmlContent) + + // Assert + + // Set the input path to what was just written and rely on the separately-tested load method to uncompress and read the contents + tool.flowXmlPath = tool.outputFlowXmlPath + assert tool.loadFlowXml() == xmlContent + } + + // TODO: Test with 128/256-bit available } public class TestAppender extends AppenderSkeleton { http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow.xml ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow.xml new file mode 100644 index 0000000..f47a883 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow.xml @@ -0,0 +1,154 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<flowController encoding-version="1.0"> + <maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount> + <maxEventDrivenThreadCount>5</maxEventDrivenThreadCount> + <rootGroup> + <id>fcf146b2-0157-1000-7850-7adf1d31e3fa</id> + <name>NiFi Flow</name> + <position x="0.0" y="0.0"/> + <comment/> + <processGroup> + <id>8a61ec1d-0158-1000-3a1a-12c54fe77838</id> + <name>EncryptedProperties Example</name> + <position x="1119.0" y="295.0"/> + <comment/> + <processor> + <id>8a621f0b-0158-1000-b5c2-92a09a124501</id> + <name>Encrypt</name> + <position x="626.0" y="237.0"/> + <styles/> + <comment/> + <class>org.apache.nifi.processors.standard.EncryptContent</class> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>STOPPED</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <runDurationNanos>0</runDurationNanos> + <property> + <name>Mode</name> + <value>Encrypt</value> + </property> + <property> + <name>key-derivation-function</name> + <value>NIFI_LEGACY</value> + </property> + <property> + <name>Encryption Algorithm</name> + <value>MD5_128AES</value> + </property> + <property> + <name>allow-weak-crypto</name> + <value>not-allowed</value> + </property> + <property> + <name>Password</name> + <value>enc{2032416987A00D9FCD757528D7AE609D7E793CA5F956641DB53E14CDB9BFCD4037B73AC705CD3F5C1C1BDE18B8D7B281}</value> + </property> + <property> + <name>raw-key-hex</name> + </property> + <property> + <name>public-keyring-file</name> + </property> + <property> + <name>public-key-user-id</name> + </property> + <property> + <name>private-keyring-file</name> + </property> + <property> + <name>private-keyring-passphrase</name> + </property> + </processor> + <processor> + <id>8a6314ee-0158-1000-6dd0-60f153db26c1</id> + <name>Decrypt</name> + <position x="630.0" y="482.0"/> + <styles/> + <comment/> + <class>org.apache.nifi.processors.standard.EncryptContent</class> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>STOPPED</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <runDurationNanos>0</runDurationNanos> + <property> + <name>Mode</name> + <value>Decrypt</value> + </property> + <property> + <name>key-derivation-function</name> + <value>NIFI_LEGACY</value> + </property> + <property> + <name>Encryption Algorithm</name> + <value>MD5_128AES</value> + </property> + <property> + <name>allow-weak-crypto</name> + <value>not-allowed</value> + </property> + <property> + <name>Password</name> + <value>enc{4B580C55B8355FE57A599B31B3B2ACA77429DBF6887C177417624026469E895F0C89FB0C9D9C64C5B2AD943035689C9C}</value> + </property> + <property> + <name>raw-key-hex</name> + </property> + <property> + <name>public-keyring-file</name> + </property> + <property> + <name>public-key-user-id</name> + </property> + <property> + <name>private-keyring-file</name> + </property> + <property> + <name>private-keyring-passphrase</name> + </property> + </processor> + <connection> + <id>8a636069-0158-1000-50ff-244f2a8eeb7a</id> + <name/> + <bendPoints/> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>8a621f0b-0158-1000-b5c2-92a09a124501</sourceId> + <sourceGroupId>8a61ec1d-0158-1000-3a1a-12c54fe77838</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>8a6314ee-0158-1000-6dd0-60f153db26c1</destinationId> + <destinationGroupId>8a61ec1d-0158-1000-3a1a-12c54fe77838</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + </connection> + </processGroup> + </rootGroup> + <controllerServices/> + <reportingTasks/> +</flowController> http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow.xml.gz ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow.xml.gz b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow.xml.gz new file mode 100644 index 0000000..5ebf096 Binary files /dev/null and b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow.xml.gz differ http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow_default_key.xml ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow_default_key.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow_default_key.xml new file mode 100644 index 0000000..f47a883 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow_default_key.xml @@ -0,0 +1,154 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<flowController encoding-version="1.0"> + <maxTimerDrivenThreadCount>10</maxTimerDrivenThreadCount> + <maxEventDrivenThreadCount>5</maxEventDrivenThreadCount> + <rootGroup> + <id>fcf146b2-0157-1000-7850-7adf1d31e3fa</id> + <name>NiFi Flow</name> + <position x="0.0" y="0.0"/> + <comment/> + <processGroup> + <id>8a61ec1d-0158-1000-3a1a-12c54fe77838</id> + <name>EncryptedProperties Example</name> + <position x="1119.0" y="295.0"/> + <comment/> + <processor> + <id>8a621f0b-0158-1000-b5c2-92a09a124501</id> + <name>Encrypt</name> + <position x="626.0" y="237.0"/> + <styles/> + <comment/> + <class>org.apache.nifi.processors.standard.EncryptContent</class> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>STOPPED</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <runDurationNanos>0</runDurationNanos> + <property> + <name>Mode</name> + <value>Encrypt</value> + </property> + <property> + <name>key-derivation-function</name> + <value>NIFI_LEGACY</value> + </property> + <property> + <name>Encryption Algorithm</name> + <value>MD5_128AES</value> + </property> + <property> + <name>allow-weak-crypto</name> + <value>not-allowed</value> + </property> + <property> + <name>Password</name> + <value>enc{2032416987A00D9FCD757528D7AE609D7E793CA5F956641DB53E14CDB9BFCD4037B73AC705CD3F5C1C1BDE18B8D7B281}</value> + </property> + <property> + <name>raw-key-hex</name> + </property> + <property> + <name>public-keyring-file</name> + </property> + <property> + <name>public-key-user-id</name> + </property> + <property> + <name>private-keyring-file</name> + </property> + <property> + <name>private-keyring-passphrase</name> + </property> + </processor> + <processor> + <id>8a6314ee-0158-1000-6dd0-60f153db26c1</id> + <name>Decrypt</name> + <position x="630.0" y="482.0"/> + <styles/> + <comment/> + <class>org.apache.nifi.processors.standard.EncryptContent</class> + <maxConcurrentTasks>1</maxConcurrentTasks> + <schedulingPeriod>0 sec</schedulingPeriod> + <penalizationPeriod>30 sec</penalizationPeriod> + <yieldPeriod>1 sec</yieldPeriod> + <bulletinLevel>WARN</bulletinLevel> + <lossTolerant>false</lossTolerant> + <scheduledState>STOPPED</scheduledState> + <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy> + <runDurationNanos>0</runDurationNanos> + <property> + <name>Mode</name> + <value>Decrypt</value> + </property> + <property> + <name>key-derivation-function</name> + <value>NIFI_LEGACY</value> + </property> + <property> + <name>Encryption Algorithm</name> + <value>MD5_128AES</value> + </property> + <property> + <name>allow-weak-crypto</name> + <value>not-allowed</value> + </property> + <property> + <name>Password</name> + <value>enc{4B580C55B8355FE57A599B31B3B2ACA77429DBF6887C177417624026469E895F0C89FB0C9D9C64C5B2AD943035689C9C}</value> + </property> + <property> + <name>raw-key-hex</name> + </property> + <property> + <name>public-keyring-file</name> + </property> + <property> + <name>public-key-user-id</name> + </property> + <property> + <name>private-keyring-file</name> + </property> + <property> + <name>private-keyring-passphrase</name> + </property> + </processor> + <connection> + <id>8a636069-0158-1000-50ff-244f2a8eeb7a</id> + <name/> + <bendPoints/> + <labelIndex>1</labelIndex> + <zIndex>0</zIndex> + <sourceId>8a621f0b-0158-1000-b5c2-92a09a124501</sourceId> + <sourceGroupId>8a61ec1d-0158-1000-3a1a-12c54fe77838</sourceGroupId> + <sourceType>PROCESSOR</sourceType> + <destinationId>8a6314ee-0158-1000-6dd0-60f153db26c1</destinationId> + <destinationGroupId>8a61ec1d-0158-1000-3a1a-12c54fe77838</destinationGroupId> + <destinationType>PROCESSOR</destinationType> + <relationship>success</relationship> + <maxWorkQueueSize>10000</maxWorkQueueSize> + <maxWorkQueueDataSize>1 GB</maxWorkQueueDataSize> + <flowFileExpiration>0 sec</flowFileExpiration> + </connection> + </processGroup> + </rootGroup> + <controllerServices/> + <reportingTasks/> +</flowController> http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow_default_key.xml.gz ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow_default_key.xml.gz b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow_default_key.xml.gz new file mode 100644 index 0000000..bb7c931 Binary files /dev/null and b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/flow_default_key.xml.gz differ http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_default.properties ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_default.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_default.properties new file mode 100644 index 0000000..1de9971 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_default.properties @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port= +nifi.web.https.host=nifi.nifi.apache.org +nifi.web.https.port=8443 +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key= +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.needClientAuth= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_protected_aes_password_128.properties ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_protected_aes_password_128.properties b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_protected_aes_password_128.properties new file mode 100644 index 0000000..a210c17 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/test/resources/nifi_with_few_sensitive_properties_protected_aes_password_128.properties @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.version=nifi-test 3.0.0 + +# security properties # +nifi.sensitive.props.key=UXcrW8T1UKAPJeun||ezUJSp30AvKGsRxJOOXoPUtZonv56Lx1 +nifi.sensitive.props.key.protected=aes/gcm/128 +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL +nifi.sensitive.props.provider=BC +nifi.sensitive.props.additional.keys= + +nifi.security.keystore=/path/to/keystore.jks +nifi.security.keystoreType=JKS +nifi.security.keystorePasswd=vp9C9a8KbSYZFdUM||RoZHB1J+sRgCKG2vKBviOn71tdhsDYH42No+VmIaFMolrTMD/zmwcKev +nifi.security.keystorePasswd.protected=aes/gcm/128 +nifi.security.keyPasswd=ttiSNTC7PUf2Hla7||W2OmVFn4bfB2ZJNBG55SaLneQeZahhF6GIaLZU+i4zRFfxKnlQ +nifi.security.keyPasswd.protected=aes/gcm/128 +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd=
