NIFI-3024 Added key migration for sensitive processor properties contained in flow.xml.gz. (nifi.sensitive.props.key)
This closes #1261. Signed-off-by: Andy LoPresto <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/2c371453 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/2c371453 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/2c371453 Branch: refs/heads/master Commit: 2c3714536fc516fc1712dac2a7a9ccd855e6f25b Parents: cdb9b81 Author: Andy LoPresto <[email protected]> Authored: Mon Nov 21 21:19:18 2016 -0800 Committer: Andy LoPresto <[email protected]> Committed: Wed Nov 23 13:26:18 2016 -0800 ---------------------------------------------------------------------- .../src/main/asciidoc/administration-guide.adoc | 32 +- .../nifi/properties/NiFiPropertiesLoader.java | 64 +- .../properties/ProtectedNiFiProperties.java | 2 +- .../AESSensitivePropertyProviderTest.groovy | 2 +- .../nifi-toolkit-encrypt-config/pom.xml | 12 + .../nifi/properties/ConfigEncryptionTool.groovy | 483 ++++++++- .../properties/ConfigEncryptionToolTest.groovy | 981 ++++++++++++++++++- .../src/test/resources/flow.xml | 154 +++ .../src/test/resources/flow.xml.gz | Bin 0 -> 1674 bytes .../src/test/resources/flow_default_key.xml | 154 +++ .../src/test/resources/flow_default_key.xml.gz | Bin 0 -> 1686 bytes .../src/test/resources/nifi_default.properties | 125 +++ ...erties_protected_aes_password_128.properties | 34 + 13 files changed, 1960 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-docs/src/main/asciidoc/administration-guide.adoc ---------------------------------------------------------------------- diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc b/nifi-docs/src/main/asciidoc/administration-guide.adoc index 2d63a9a..287760a 100644 --- a/nifi-docs/src/main/asciidoc/administration-guide.adoc +++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc @@ -1002,19 +1002,25 @@ The default encryption algorithm utilized is AES/GCM 128/256-bit. 128-bit is use You can use the following command line options with the `encrypt-config` tool: -* `-b,--bootstrapConf <arg>` The bootstrap.conf file to persist master key -* `-e,--oldKey <arg>` The old raw hexadecimal key to use during key migration -* `-h,--help` Prints this usage message -* `-k,--key <arg>` The raw hexadecimal key to use to encrypt the sensitive properties -* `-m,--migrate` If provided, the sensitive properties will be re-encrypted with a new key -* `-n,--niFiProperties <arg>` The nifi.properties file containing unprotected config values (will be overwritten) -* `-o,--outputNiFiProperties <arg>` The destination nifi.properties file containing protected config values (will not modify input nifi.properties) -* `-p,--password <arg>` The password from which to derive the key to use to encrypt the sensitive properties -* `-r,--useRawKey` If provided, the secure console will prompt for the raw key value in hexadecimal form -* `-v,--verbose` Sets verbose mode (default false) -* `-w,--oldPassword <arg>` The old password from which to derive the key during migration -* `-l,--loginIdentityProviders <arg>` The login-identity-providers.xml file containing unprotected config values (will be overwritten) -* `-i,--outputLoginIdentityProviders <arg>` The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml) + * `-A`,`--newFlowAlgorithm <arg>` The algorithm to use to encrypt the sensitive processor properties in flow.xml.gz + * `-b`,`--bootstrapConf <arg>` The bootstrap.conf file to persist master key + * `-e`,`--oldKey <arg>` The old raw hexadecimal key to use during key migration + * `-f`,`--flowXml <arg>` The flow.xml.gz file currently protected with old password (will be overwritten) + * `-g`,`--outputFlowXml <arg>` The destination flow.xml.gz file containing protected config values (will not modify input flow.xml.gz) + * `-h`,`--help` Prints this usage message + * `-i`,`--outputLoginIdentityProviders <arg>` The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml) + * `-k`,`--key <arg>` The raw hexadecimal key to use to encrypt the sensitive properties + * `-l`,`--loginIdentityProviders <arg>` The login-identity-providers.xml file containing unprotected config values (will be overwritten) + * `-m`,`--migrate` If provided, the nifi.properties and/or login-identity-providers.xml sensitive properties will be re-encrypted with a new key + * `-n`,`--niFiProperties <arg>` The nifi.properties file containing unprotected config values (will be overwritten) + * `-o`,`--outputNiFiProperties <arg>` The destination nifi.properties file containing protected config values (will not modify input nifi.properties) + * `-p`,`--password <arg>` The password from which to derive the key to use to encrypt the sensitive properties + * `-P`,`--newFlowProvider <arg>` The security provider to use to encrypt the sensitive processor properties in flow.xml.gz + * `-r`,`--useRawKey` If provided, the secure console will prompt for the raw key value in hexadecimal form + * `-s`,`--propsKey <arg>` The password or key to use to encrypt the sensitive processor properties in flow.xml.gz + * `-v`,`--verbose` Sets verbose mode (default false) + * `-w`,`--oldPassword <arg>` The old password from which to derive the key during migration + * `-x`,`--encryptFlowXmlOnly` If provided, the properties in flow.xml.gz will be re-encrypted with a new key but the nifi.properties and/or login-identity-providers.xml files will not be modified As an example of how the tool works, assume that you have installed the tool on a machine supporting 256-bit encryption and with the following existing values in the 'nifi.properties' file: http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java index 20b5191..b9230c3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/NiFiPropertiesLoader.java @@ -29,6 +29,7 @@ import java.util.Optional; import java.util.Properties; import java.util.stream.Stream; import javax.crypto.Cipher; +import org.apache.commons.lang3.StringUtils; import org.apache.nifi.util.NiFiProperties; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.slf4j.Logger; @@ -53,7 +54,7 @@ public class NiFiPropertiesLoader { /** * Returns an instance of the loader configured with the key. - * + * <p> * <p> * NOTE: This method is used reflectively by the process which starts NiFi * so changes to it must be made in conjunction with that mechanism.</p> @@ -109,31 +110,48 @@ public class NiFiPropertiesLoader { * @throws IOException if the file is not readable */ public static String extractKeyFromBootstrapFile() throws IOException { - // Guess at location of bootstrap.conf file from nifi.properties file - String defaultNiFiPropertiesPath = getDefaultFilePath(); - File propertiesFile = new File(defaultNiFiPropertiesPath); - File confDir = new File(propertiesFile.getParent()); - if (confDir.exists() && confDir.canRead()) { - File expectedBootstrapFile = new File(confDir, "bootstrap.conf"); - if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) { - try (Stream<String> stream = Files.lines(Paths.get(expectedBootstrapFile.getAbsolutePath()))) { - Optional<String> keyLine = stream.filter(l -> l.startsWith(BOOTSTRAP_KEY_PREFIX)).findFirst(); - if (keyLine.isPresent()) { - return keyLine.get().split("=", 2)[1]; - } else { - logger.warn("No encryption key present in the bootstrap.conf file at {}", expectedBootstrapFile.getAbsolutePath()); - return ""; - } - } catch (IOException e) { - logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", expectedBootstrapFile.getAbsolutePath()); - throw new IOException("Cannot read from bootstrap.conf", e); - } + return extractKeyFromBootstrapFile(""); + } + + /** + * Returns the key (if any) used to encrypt sensitive properties, extracted from {@code $NIFI_HOME/conf/bootstrap.conf}. + * + * @param bootstrapPath the path to the bootstrap file + * @return the key in hexadecimal format + * @throws IOException if the file is not readable + */ + public static String extractKeyFromBootstrapFile(String bootstrapPath) throws IOException { + File expectedBootstrapFile; + if (StringUtils.isBlank(bootstrapPath)) { + // Guess at location of bootstrap.conf file from nifi.properties file + String defaultNiFiPropertiesPath = getDefaultFilePath(); + File propertiesFile = new File(defaultNiFiPropertiesPath); + File confDir = new File(propertiesFile.getParent()); + if (confDir.exists() && confDir.canRead()) { + expectedBootstrapFile = new File(confDir, "bootstrap.conf"); } else { - logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath()); + logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- conf/ directory is missing or permissions are incorrect", confDir.getAbsolutePath()); throw new IOException("Cannot read from bootstrap.conf"); } } else { - logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- conf/ directory is missing or permissions are incorrect", confDir.getAbsolutePath()); + expectedBootstrapFile = new File(bootstrapPath); + } + + if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead()) { + try (Stream<String> stream = Files.lines(Paths.get(expectedBootstrapFile.getAbsolutePath()))) { + Optional<String> keyLine = stream.filter(l -> l.startsWith(BOOTSTRAP_KEY_PREFIX)).findFirst(); + if (keyLine.isPresent()) { + return keyLine.get().split("=", 2)[1]; + } else { + logger.warn("No encryption key present in the bootstrap.conf file at {}", expectedBootstrapFile.getAbsolutePath()); + return ""; + } + } catch (IOException e) { + logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key", expectedBootstrapFile.getAbsolutePath()); + throw new IOException("Cannot read from bootstrap.conf", e); + } + } else { + logger.error("Cannot read from bootstrap.conf file at {} to extract encryption key -- file is missing or permissions are incorrect", expectedBootstrapFile.getAbsolutePath()); throw new IOException("Cannot read from bootstrap.conf"); } } @@ -254,7 +272,7 @@ public class NiFiPropertiesLoader { /** * Returns the loaded {@link NiFiProperties} instance. If none is currently * loaded, attempts to load the default instance. - * + * <p> * <p> * NOTE: This method is used reflectively by the process which starts NiFi * so changes to it must be made in conjunction with that mechanism.</p> http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java index 83320a0..4774dc7 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/main/java/org/apache/nifi/properties/ProtectedNiFiProperties.java @@ -284,7 +284,7 @@ class ProtectedNiFiProperties extends StandardNiFiProperties { * @param key the key identifying the sensitive property * @return the key identifying the protection scheme for the sensitive property */ - public String getProtectionKey(String key) { + public static String getProtectionKey(String key) { if (key == null || key.isEmpty()) { throw new IllegalArgumentException("Cannot find protection key for null key"); } http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy index 4c5a34d..7896afe 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/AESSensitivePropertyProviderTest.groovy @@ -447,7 +447,7 @@ class AESSensitivePropertyProviderTest extends GroovyTestCase { @Test public void testShouldEncryptArbitraryValues() { // Arrange - def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message"] + def values = ["thisIsABadPassword", "thisIsABadSensitiveKeyPassword", "thisIsABadKeystorePassword", "thisIsABadKeyPassword", "thisIsABadTruststorePassword", "This is an encrypted banner message", "nififtw!"] String key = "2C576A9585DB862F5ECBEE5B4FFFCCA1" //getKeyOfSize(128) // key = "0" * 64 http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml index f2ef7e9..22e83f4 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml @@ -58,6 +58,18 @@ <version>1.16.0</version> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.nifi</groupId> + <artifactId>nifi-framework-core</artifactId> + <version>1.1.0-SNAPSHOT</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/nifi/blob/2c371453/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy index 0f69d1b..71166f0 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy @@ -25,6 +25,7 @@ import org.apache.commons.cli.HelpFormatter import org.apache.commons.cli.Options import org.apache.commons.cli.ParseException import org.apache.commons.codec.binary.Hex +import org.apache.commons.io.IOUtils import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException import org.apache.nifi.toolkit.tls.commandLine.ExitCode import org.apache.nifi.util.NiFiProperties @@ -37,9 +38,16 @@ import org.slf4j.LoggerFactory import org.xml.sax.SAXException import javax.crypto.Cipher +import javax.crypto.SecretKey +import javax.crypto.SecretKeyFactory +import javax.crypto.spec.PBEKeySpec +import javax.crypto.spec.PBEParameterSpec import java.nio.charset.StandardCharsets import java.security.KeyException +import java.security.SecureRandom import java.security.Security +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream class ConfigEncryptionTool { private static final Logger logger = LoggerFactory.getLogger(ConfigEncryptionTool.class) @@ -49,14 +57,23 @@ class ConfigEncryptionTool { public String outputNiFiPropertiesPath public String loginIdentityProvidersPath public String outputLoginIdentityProvidersPath + public String flowXmlPath + public String outputFlowXmlPath private String keyHex private String migrationKeyHex private String password private String migrationPassword + // This is the raw value used in nifi.sensitive.props.key + private String flowPropertiesPassword + + private String newFlowAlgorithm + private String newFlowProvider + private NiFiProperties niFiProperties private String loginIdentityProviders + private String flowXml private boolean usingPassword = true private boolean usingPasswordMigration = true @@ -64,6 +81,8 @@ class ConfigEncryptionTool { private boolean isVerbose = false private boolean handlingNiFiProperties = false private boolean handlingLoginIdentityProviders = false + private boolean handlingFlowXml = false + private boolean ignorePropertiesFiles = false private static final String HELP_ARG = "help" private static final String VERBOSE_ARG = "verbose" @@ -72,13 +91,21 @@ class ConfigEncryptionTool { private static final String LOGIN_IDENTITY_PROVIDERS_ARG = "loginIdentityProviders" private static final String OUTPUT_NIFI_PROPERTIES_ARG = "outputNiFiProperties" private static final String OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG = "outputLoginIdentityProviders" + private static final String FLOW_XML_ARG = "flowXml" + private static final String OUTPUT_FLOW_XML_ARG = "outputFlowXml" private static final String KEY_ARG = "key" private static final String PASSWORD_ARG = "password" private static final String KEY_MIGRATION_ARG = "oldKey" private static final String PASSWORD_MIGRATION_ARG = "oldPassword" private static final String USE_KEY_ARG = "useRawKey" private static final String MIGRATION_ARG = "migrate" + private static final String PROPS_KEY_ARG = "propsKey" + private static final String DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG = "encryptFlowXmlOnly" + private static final String NEW_FLOW_ALGORITHM_ARG = "newFlowAlgorithm" + private static final String NEW_FLOW_PROVIDER_ARG = "newFlowProvider" + // Hard-coded fallback value from {@link org.apache.nifi.encrypt.StringEncryptor} + private static final String DEFAULT_NIFI_SENSITIVE_PROPS_KEY = "nififtw!" private static final int MIN_PASSWORD_LENGTH = 12 // Strong parameters as of 12 Aug 2016 @@ -86,6 +113,10 @@ class ConfigEncryptionTool { private static final int SCRYPT_R = 8 private static final int SCRYPT_P = 1 + // Hard-coded values from StandardPBEByteEncryptor which will be removed during refactor of all flow encryption code in NIFI-1465 + private static final int DEFAULT_KDF_ITERATIONS = 1000 + private static final int DEFAULT_SALT_SIZE_BYTES = 16 + private static final String BOOTSTRAP_KEY_COMMENT = "# Master key in hexadecimal format for encrypted sensitive configuration values" private static final String BOOTSTRAP_KEY_PREFIX = "nifi.bootstrap.sensitive.key=" @@ -96,10 +127,20 @@ class ConfigEncryptionTool { private static final String FOOTER = buildFooter() private static - final String DEFAULT_DESCRIPTION = "This tool reads from a nifi.properties and/or login-identity-providers.xml file with plain sensitive configuration values, prompts the user for a master key, and encrypts each value. It will replace the plain value with the protected value in the same file (or write to a new file if specified)." + final String DEFAULT_DESCRIPTION = "This tool reads from a nifi.properties and/or " + + "login-identity-providers.xml file with plain sensitive configuration values, " + + "prompts the user for a master key, and encrypts each value. It will replace the " + + "plain value with the protected value in the same file (or write to a new file if " + + "specified). It can also be used to migrate already-encrypted values in those " + + "files or in flow.xml.gz to be encrypted with a new key." private static final String LDAP_PROVIDER_CLASS = "org.apache.nifi.ldap.LdapProvider" - static private final String LDAP_PROVIDER_REGEX = /<provider>[\s\S]*?<class>\s*org\.apache\.nifi\.ldap\.LdapProvider[\s\S]*?<\/provider>/ - static private final String XML_DECLARATION_REGEX = /<\?xml version="1.0" encoding="UTF-8"\?>/ + private static + final String LDAP_PROVIDER_REGEX = /<provider>[\s\S]*?<class>\s*org\.apache\.nifi\.ldap\.LdapProvider[\s\S]*?<\/provider>/ + private static final String XML_DECLARATION_REGEX = /<\?xml version="1.0" encoding="UTF-8"\?>/ + private static final String WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX = /enc\{[a-fA-F0-9]+?\}/ + + private static final String DEFAULT_PROVIDER = BouncyCastleProvider.PROVIDER_NAME + private static final String DEFAULT_FLOW_ALGORITHM = "PBEWITHMD5AND256BITAES-CBC-OPENSSL" private static String buildHeader(String description = DEFAULT_DESCRIPTION) { "${SEP}${description}${SEP * 2}" @@ -109,8 +150,8 @@ class ConfigEncryptionTool { "${SEP}Java home: ${System.getenv(JAVA_HOME)}${SEP}NiFi Toolkit home: ${System.getenv(NIFI_TOOLKIT_HOME)}" } - private final Options options; - private final String header; + private final Options options + private final String header public ConfigEncryptionTool() { @@ -124,15 +165,21 @@ class ConfigEncryptionTool { options.addOption("v", VERBOSE_ARG, false, "Sets verbose mode (default false)") options.addOption("n", NIFI_PROPERTIES_ARG, true, "The nifi.properties file containing unprotected config values (will be overwritten)") options.addOption("l", LOGIN_IDENTITY_PROVIDERS_ARG, true, "The login-identity-providers.xml file containing unprotected config values (will be overwritten)") + options.addOption("f", FLOW_XML_ARG, true, "The flow.xml.gz file currently protected with old password (will be overwritten)") options.addOption("b", BOOTSTRAP_CONF_ARG, true, "The bootstrap.conf file to persist master key") options.addOption("o", OUTPUT_NIFI_PROPERTIES_ARG, true, "The destination nifi.properties file containing protected config values (will not modify input nifi.properties)") options.addOption("i", OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG, true, "The destination login-identity-providers.xml file containing protected config values (will not modify input login-identity-providers.xml)") + options.addOption("g", OUTPUT_FLOW_XML_ARG, true, "The destination flow.xml.gz file containing protected config values (will not modify input flow.xml.gz)") options.addOption("k", KEY_ARG, true, "The raw hexadecimal key to use to encrypt the sensitive properties") options.addOption("e", KEY_MIGRATION_ARG, true, "The old raw hexadecimal key to use during key migration") options.addOption("p", PASSWORD_ARG, true, "The password from which to derive the key to use to encrypt the sensitive properties") options.addOption("w", PASSWORD_MIGRATION_ARG, true, "The old password from which to derive the key during migration") options.addOption("r", USE_KEY_ARG, false, "If provided, the secure console will prompt for the raw key value in hexadecimal form") - options.addOption("m", MIGRATION_ARG, false, "If provided, the sensitive properties will be re-encrypted with a new key") + options.addOption("m", MIGRATION_ARG, false, "If provided, the nifi.properties and/or login-identity-providers.xml sensitive properties will be re-encrypted with a new key") + options.addOption("x", DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG, false, "If provided, the properties in flow.xml.gz will be re-encrypted with a new key but the nifi.properties and/or login-identity-providers.xml files will not be modified") + options.addOption("s", PROPS_KEY_ARG, true, "The password or key to use to encrypt the sensitive processor properties in flow.xml.gz") + options.addOption("A", NEW_FLOW_ALGORITHM_ARG, true, "The algorithm to use to encrypt the sensitive processor properties in flow.xml.gz") + options.addOption("P", NEW_FLOW_PROVIDER_ARG, true, "The security provider to use to encrypt the sensitive processor properties in flow.xml.gz") } /** @@ -169,13 +216,36 @@ class ConfigEncryptionTool { bootstrapConfPath = commandLine.getOptionValue(BOOTSTRAP_CONF_ARG) + // If this flag is provided, the nifi.properties is necessary to read/write the flow encryption key, but the encryption process will not actually be applied to nifi.properties / login-identity-providers.xml + if (commandLine.hasOption(DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG)) { + handlingNiFiProperties = false + handlingLoginIdentityProviders = false + ignorePropertiesFiles = true + } else { + if (commandLine.hasOption(LOGIN_IDENTITY_PROVIDERS_ARG)) { + if (isVerbose) { + logger.info("Handling encryption of login-identity-providers.xml") + } + loginIdentityProvidersPath = commandLine.getOptionValue(LOGIN_IDENTITY_PROVIDERS_ARG) + outputLoginIdentityProvidersPath = commandLine.getOptionValue(OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG, loginIdentityProvidersPath) + handlingLoginIdentityProviders = true + + if (loginIdentityProvidersPath == outputLoginIdentityProvidersPath) { + // TODO: Add confirmation pause and provide -y flag to offer no-interaction mode? + logger.warn("The source login-identity-providers.xml and destination login-identity-providers.xml are identical [${outputLoginIdentityProvidersPath}] so the original will be overwritten") + } + } + } + + // This needs to occur even if the nifi.properties won't be encrypted if (commandLine.hasOption(NIFI_PROPERTIES_ARG)) { - if (isVerbose) { + boolean ignoreFlagPresent = commandLine.hasOption(DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG) + if (isVerbose && !ignoreFlagPresent) { logger.info("Handling encryption of nifi.properties") } niFiPropertiesPath = commandLine.getOptionValue(NIFI_PROPERTIES_ARG) outputNiFiPropertiesPath = commandLine.getOptionValue(OUTPUT_NIFI_PROPERTIES_ARG, niFiPropertiesPath) - handlingNiFiProperties = true + handlingNiFiProperties = !ignoreFlagPresent if (niFiPropertiesPath == outputNiFiPropertiesPath) { // TODO: Add confirmation pause and provide -y flag to offer no-interaction mode? @@ -183,17 +253,24 @@ class ConfigEncryptionTool { } } - if (commandLine.hasOption(LOGIN_IDENTITY_PROVIDERS_ARG)) { + if (commandLine.hasOption(FLOW_XML_ARG)) { if (isVerbose) { - logger.info("Handling encryption of login-identity-providers.xml") + logger.info("Handling encryption of flow.xml.gz") } - loginIdentityProvidersPath = commandLine.getOptionValue(LOGIN_IDENTITY_PROVIDERS_ARG) - outputLoginIdentityProvidersPath = commandLine.getOptionValue(OUTPUT_LOGIN_IDENTITY_PROVIDERS_ARG, loginIdentityProvidersPath) - handlingLoginIdentityProviders = true + flowXmlPath = commandLine.getOptionValue(FLOW_XML_ARG) + outputFlowXmlPath = commandLine.getOptionValue(OUTPUT_FLOW_XML_ARG, flowXmlPath) + handlingFlowXml = true + + newFlowAlgorithm = commandLine.getOptionValue(NEW_FLOW_ALGORITHM_ARG) + newFlowProvider = commandLine.getOptionValue(NEW_FLOW_PROVIDER_ARG) - if (loginIdentityProvidersPath == outputLoginIdentityProvidersPath) { + if (flowXmlPath == outputFlowXmlPath) { // TODO: Add confirmation pause and provide -y flag to offer no-interaction mode? - logger.warn("The source login-identity-providers.xml and destination login-identity-providers.xml are identical [${outputLoginIdentityProvidersPath}] so the original will be overwritten") + logger.warn("The source flow.xml.gz and destination flow.xml.gz are identical [${outputFlowXmlPath}] so the original will be overwritten") + } + + if (!commandLine.hasOption(NIFI_PROPERTIES_ARG)) { + printUsageAndThrow("In order to migrate a flow.xml.gz, a nifi.properties file must also be specified via '-n'/'--${NIFI_PROPERTIES_ARG}'.", ExitCode.INVALID_ARGS) } } @@ -203,6 +280,8 @@ class ConfigEncryptionTool { logger.info("(dest) nifi.properties: \t${outputNiFiPropertiesPath}") logger.info("(src) login-identity-providers.xml: \t${loginIdentityProvidersPath}") logger.info("(dest) login-identity-providers.xml: \t${outputLoginIdentityProvidersPath}") + logger.info("(src) flow.xml.gz: \t\t\t\t\t${flowXmlPath}") + logger.info("(dest) flow.xml.gz: \t\t\t\t\t${outputFlowXmlPath}") } // TODO: Implement in NIFI-2655 @@ -251,6 +330,10 @@ class ConfigEncryptionTool { usingPassword = false } } + + if (commandLine.hasOption(PROPS_KEY_ARG)) { + flowPropertiesPassword = commandLine.getOptionValue(PROPS_KEY_ARG) + } } catch (ParseException e) { if (isVerbose) { logger.error("Encountered an error", e) @@ -299,6 +382,10 @@ class ConfigEncryptionTool { getKeyInternal(TextDevices.defaultTextDevice(), migrationKeyHex, migrationPassword, usingPasswordMigration) } + private String getFlowPassword(TextDevice textDevice = TextDevices.defaultTextDevice()) { + readPasswordFromConsole(textDevice) + } + private static String readKeyFromConsole(TextDevice textDevice) { textDevice.printf("Enter the master key in hexadecimal format (spaces acceptable): ") new String(textDevice.readPassword()) @@ -387,6 +474,227 @@ class ConfigEncryptionTool { } } + /** + * Loads the flow definition from the provided file path, handling the GZIP file compression. Unlike {@link #loadLoginIdentityProviders()} this method does not decrypt the content (for performance and separation of concern reasons). + * + * @return the file content + * @throw IOException if the flow.xml.gz file cannot be read + */ + private String loadFlowXml() throws IOException { + File flowXmlFile + if (flowXmlPath && (flowXmlFile = new File(flowXmlPath)).exists()) { + try { + new FileInputStream(flowXmlPath).withCloseable { + new GZIPInputStream(it).withCloseable { + String xmlContent = IOUtils.toString(it, StandardCharsets.UTF_8) + return xmlContent + } + } + } catch (RuntimeException e) { + if (isVerbose) { + logger.error("Encountered an error", e) + } + throw new IOException("Cannot load flow from [${flowXmlPath}]", e) + } + } else { + printUsageAndThrow("Cannot load flow from [${flowXmlPath}]", ExitCode.ERROR_READING_NIFI_PROPERTIES) + } + } + + /** + * Decrypts a single element encrypted in the flow.xml.gz style (hex-encoded and wrapped with "enc{" and "}"). + * + * Example: + * {@code enc{0123456789ABCDEF} } -> "some text" + * + * @param wrappedCipherText the wrapped and hex-encoded cipher text + * @param password the password used to encrypt the content (UTF-8 encoded) + * @param algorithm the encryption and KDF algorithm (defaults to PBEWITHMD5AND256BITAES-CBC-OPENSSL) + * @param provider the security provider (defaults to BC) + * @return the plaintext in UTF-8 encoding + */ + private + static String decryptFlowElement(String wrappedCipherText, String password, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) { + // Drop the "enc{" and closing "}" + if (!(wrappedCipherText =~ WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX)) { + throw new SensitivePropertyProtectionException("The provided cipher text does not match the expected format 'enc{0123456789ABCDEF...}'") + } + String unwrappedCipherText = wrappedCipherText.replaceAll(/enc\{/, "")[0..<-1] + if (unwrappedCipherText.length() % 2 == 1 || unwrappedCipherText.length() == 0) { + throw new SensitivePropertyProtectionException("The provided cipher text must have an even number of hex characters") + } + + // Decode the hex + byte[] cipherBytes = Hex.decodeHex(unwrappedCipherText.chars) + + /* The structure of each cipher text is 16 bytes of salt || actual cipher text, + * so extract the salt (32 bytes encoded as hex, 16 bytes raw) and combine that + * with the default (and unchanged) iteration count that is hardcoded in + * {@link StandardPBEByteEncryptor}. I am extracting + * these values to magic numbers here so when the refactoring is performed, + * stronger decisions can be implemented here + */ + byte[] saltBytes = cipherBytes[0..<DEFAULT_SALT_SIZE_BYTES] + cipherBytes = cipherBytes[DEFAULT_SALT_SIZE_BYTES..-1] + + Cipher decryptionCipher = generateFlowDecryptionCipher(password, saltBytes, algorithm, provider) + + byte[] plainBytes = decryptionCipher.doFinal(cipherBytes) + new String(plainBytes, StandardCharsets.UTF_8) + } + + /** + * Returns an initialized {@link javax.crypto.Cipher} instance with the extracted salt. + * + * @param password the password (UTF-8 encoding) + * @param saltBytes the salt (raw bytes) + * @param algorithm the KDF/encryption algorithm + * @param provider the security provider + * @return the initialized {@link javax.crypto.Cipher} + */ + private + static Cipher generateFlowDecryptionCipher(String password, byte[] saltBytes, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) { + Cipher decryptCipher = Cipher.getInstance(algorithm, provider) + PBEKeySpec keySpec = new PBEKeySpec(password.chars) + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(algorithm, provider) + SecretKey pbeKey = keyFactory.generateSecret(keySpec) + PBEParameterSpec parameterSpec = new PBEParameterSpec(saltBytes, DEFAULT_KDF_ITERATIONS) + decryptCipher.init(Cipher.DECRYPT_MODE, pbeKey, parameterSpec) + decryptCipher + } + + /** + * Encrypts a single element in the flow.xml.gz style (hex-encoded and wrapped with "enc{" and "}"). + * + * Example: + * "some text" -> {@code enc{0123456789ABCDEF} } + * + * @param plaintext the plaintext in UTF-8 encoding + * @param saltBytes the salt to embed in the cipher text to allow key derivation and decryption later in raw format + * @param encryptCipher the configured Cipher instance + * @return the wrapped and hex-encoded cipher text + */ + private static String encryptFlowElement(String plaintext, byte[] saltBytes, Cipher encryptCipher) { + byte[] plainBytes = plaintext?.getBytes(StandardCharsets.UTF_8) ?: new byte[0] + + /* The structure of each cipher text is 16 bytes of salt || actual cipher text, + * so extract the salt (32 bytes encoded as hex, 16 bytes raw) and combine that + * with the default (and unchanged) iteration count that is hardcoded in + * {@link StandardPBEByteEncryptor}. I am extracting + * these values to magic numbers here so when the refactoring is performed, + * stronger decisions can be implemented here + */ + if (saltBytes.length != DEFAULT_SALT_SIZE_BYTES) { + throw new SensitivePropertyProtectionException("The salt must be ${DEFAULT_SALT_SIZE_BYTES} bytes") + } + + byte[] cipherBytes = encryptCipher.doFinal(plainBytes) + byte[] saltAndCipherBytes = concatByteArrays(saltBytes, cipherBytes) + + // Encode the hex + String hexEncodedCipherText = Hex.encodeHexString(saltAndCipherBytes) + "enc{${hexEncodedCipherText}}" + } + + /** + * Utility method to quickly concatenate an arbitrary number of byte[]. + * + * @param arrays the byte[] arrays + * @returna single byte[] containing the values concatenated + */ + private static byte[] concatByteArrays(byte[] ... arrays) { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream() + arrays.each { byte[] it -> outputStream.write(it) } + outputStream.toByteArray() + } + + /** + * Scans XML content and decrypts each encrypted element, then re-encrypts it with the new key, and returns the final XML content. + * + * @param flowXmlContent the original flow.xml.gz content + * @param existingFlowPassword the existing value of nifi.sensitive.props.key (not a raw key, but rather a password) + * @param newFlowPassword the password to use to for encryption (not a raw key, but rather a password) + * @param existingAlgorithm the KDF algorithm to use (defaults to PBEWITHMD5AND256BITAES-CBC-OPENSSL) + * @param existingProvider the {@link java.security.Provider} to use (defaults to BC) + * @return the encrypted XML content + */ + private String migrateFlowXmlContent(String flowXmlContent, String existingFlowPassword, String newFlowPassword, String existingAlgorithm = DEFAULT_FLOW_ALGORITHM, String existingProvider = DEFAULT_PROVIDER, String newAlgorithm = DEFAULT_FLOW_ALGORITHM, String newProvider = DEFAULT_PROVIDER) { + /* For re-encryption, for performance reasons, we will use a fixed salt for all of + * the operations. These values are stored in the same file and the default key is in the + * source code (see NIFI-1465 and NIFI-1277), so the security trade-off is minimal + * but the performance hit is substantial. We can't make this decision for + * decryption because the FlowSerializer still uses StringEncryptor which does not + * follow this pattern + */ + byte[] encryptionSalt = new byte[DEFAULT_SALT_SIZE_BYTES] + new SecureRandom().nextBytes(encryptionSalt) + Cipher encryptCipher = generateFlowEncryptionCipher(newFlowPassword, encryptionSalt, newAlgorithm, newProvider) + + int elementCount = 0 + + // Scan the XML content and identify every encrypted element, decrypt it, and replace it with the re-encrypted value + String migratedFlowXmlContent = flowXmlContent.replaceAll(WRAPPED_FLOW_XML_CIPHER_TEXT_REGEX) { String wrappedCipherText -> + String plaintext = decryptFlowElement(wrappedCipherText, existingFlowPassword, existingAlgorithm, existingProvider) + byte[] cipherBytes = encryptCipher.doFinal(plaintext.bytes) + byte[] saltAndCipherBytes = concatByteArrays(encryptionSalt, cipherBytes) + elementCount++ + "enc{${Hex.encodeHex(saltAndCipherBytes)}}" + } + + if (isVerbose) { + logger.info("Decrypted and re-encrypted ${elementCount} elements for flow.xml.gz") + } + + migratedFlowXmlContent + } + + /** + * Returns an initialized encryption cipher for the flow.xml.gz content. + * + * @param newFlowPassword the new encryption password + * @param saltBytes the salt [16 bytes in raw format] + * @param algorithm the KDF/encryption algorithm + * @param provider the security provider + * @return the initialized cipher instance + */ + private + static Cipher generateFlowEncryptionCipher(String newFlowPassword, byte[] saltBytes, String algorithm = DEFAULT_FLOW_ALGORITHM, String provider = DEFAULT_PROVIDER) { + /* The Jasypt StringEncryptor implementation is final and has some design decisions + * that will pollute this code (i.e. using a random salt on every encrypt operation + * rather than a unique IV, so the derived key for every encrypt/decrypt operation is + * different, which is very wasteful), so just use the standard JCE ciphers with the + * password derived using the prescribed algorithm + */ + Cipher encryptCipher = Cipher.getInstance(algorithm, provider) + + /* For re-encryption, for performance reasons, we will use a fixed salt for all of + * the operations. These values are stored in the same file and the default key is in the + * source code (see NIFI-1465 and NIFI-1277), so the security trade-off is minimal + * but the performance hit is substantial. We can't make this decision for + * decryption because the FlowSerializer still uses StringEncryptor which does not + * follow this pattern + */ + PBEKeySpec keySpec = new PBEKeySpec(newFlowPassword.chars) + SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(algorithm, provider) + SecretKey pbeKey = keyFactory.generateSecret(keySpec) + PBEParameterSpec parameterSpec = new PBEParameterSpec(saltBytes, DEFAULT_KDF_ITERATIONS) + encryptCipher.init(Cipher.ENCRYPT_MODE, pbeKey, parameterSpec) + encryptCipher + } + + /** + * Writes the XML content to the {@link .outputFlowXmlPath} location, handling the GZIP file compression. + * + * @param flowXmlContent the XML content to write + */ + private void writeFlowXmlToFile(String flowXmlContent) { + new FileOutputStream(outputFlowXmlPath).withCloseable { + new GZIPOutputStream(it).withCloseable { + IOUtils.write(flowXmlContent, it, StandardCharsets.UTF_8) + } + } + } + String decryptLoginIdentityProviders(String encryptedXml, String existingKeyHex = keyHex) { AESSensitivePropertyProvider sensitivePropertyProvider = new AESSensitivePropertyProvider(existingKeyHex) @@ -654,8 +962,11 @@ class ConfigEncryptionTool { List<String> lines = originalPropertiesFile.readLines() ProtectedNiFiProperties protectedNiFiProperties = new ProtectedNiFiProperties(niFiProperties) - // Only need to replace the keys that have been protected + // Only need to replace the keys that have been protected AND nifi.sensitive.props.key Map<String, String> protectedKeys = protectedNiFiProperties.getProtectedPropertyKeys() + if (!protectedKeys.containsKey(NiFiProperties.SENSITIVE_PROPS_KEY)) { + protectedKeys.put(NiFiProperties.SENSITIVE_PROPS_KEY, protectedNiFiProperties.getProperty(ProtectedNiFiProperties.getProtectionKey(NiFiProperties.SENSITIVE_PROPS_KEY))) + } protectedKeys.each { String key, String protectionScheme -> int l = lines.findIndexOf { it.startsWith(key) } @@ -664,7 +975,7 @@ class ConfigEncryptionTool { } // Get the index of the following line (or cap at max) int p = l + 1 > lines.size() ? lines.size() : l + 1 - String protectionLine = "${protectedNiFiProperties.getProtectionKey(key)}=${protectionScheme}" + String protectionLine = "${protectedNiFiProperties.getProtectionKey(key)}=${protectionScheme ?: ""}" if (p < lines.size() && lines.get(p).startsWith("${protectedNiFiProperties.getProtectionKey(key)}=")) { lines.set(p, protectionLine) } else { @@ -765,6 +1076,28 @@ class ConfigEncryptionTool { "NIFI_SCRYPT_SALT".getBytes(StandardCharsets.UTF_8) } + private String getExistingFlowPassword() { + return niFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY) as String ?: DEFAULT_NIFI_SENSITIVE_PROPS_KEY + } + + /** + * Utility method which returns true if the {@link org.apache.nifi.util.NiFiProperties} instance has encrypted properties. + * + * @return true if the properties instance will require a key to access + */ + boolean niFiPropertiesAreEncrypted() { + if (niFiPropertiesPath) { + try { + def nfp = NiFiPropertiesLoader.withKey(keyHex).readProtectedPropertiesFromDisk(new File(niFiPropertiesPath)) + return nfp.hasProtectedKeys() + } catch (SensitivePropertyProtectionException | IOException e) { + return true + } + } else { + return false + } + } + /** * Runs main tool logic (parsing arguments, reading files, protecting properties, and writing key and properties out to destination files). * @@ -779,48 +1112,56 @@ class ConfigEncryptionTool { try { tool.parse(args) - tool.keyHex = tool.getKey() - - if (!tool.keyHex) { - tool.printUsageAndThrow("Hex key must be provided", ExitCode.INVALID_ARGS) - } - - try { - // Validate the length and format - tool.keyHex = parseKey(tool.keyHex) - } catch (KeyException e) { - if (tool.isVerbose) { - logger.error("Encountered an error", e) + boolean existingNiFiPropertiesAreEncrypted = tool.niFiPropertiesAreEncrypted() + if (!tool.ignorePropertiesFiles || (tool.handlingFlowXml && existingNiFiPropertiesAreEncrypted)) { + // If we are handling the flow.xml.gz and nifi.properties is already encrypted, try getting the key from bootstrap.conf rather than the console + if (tool.ignorePropertiesFiles) { + tool.keyHex = NiFiPropertiesLoader.extractKeyFromBootstrapFile(tool.bootstrapConfPath) + } else { + tool.keyHex = tool.getKey() } - tool.printUsageAndThrow(e.getMessage(), ExitCode.INVALID_ARGS) - } - if (tool.migration) { - String migrationKeyHex = tool.getMigrationKey() - - if (!migrationKeyHex) { - tool.printUsageAndThrow("Original hex key must be provided for migration", ExitCode.INVALID_ARGS) + if (!tool.keyHex) { + tool.printUsageAndThrow("Hex key must be provided", ExitCode.INVALID_ARGS) } try { // Validate the length and format - tool.migrationKeyHex = parseKey(migrationKeyHex) + tool.keyHex = parseKey(tool.keyHex) } catch (KeyException e) { if (tool.isVerbose) { logger.error("Encountered an error", e) } tool.printUsageAndThrow(e.getMessage(), ExitCode.INVALID_ARGS) } + + if (tool.migration) { + String migrationKeyHex = tool.getMigrationKey() + + if (!migrationKeyHex) { + tool.printUsageAndThrow("Original hex key must be provided for migration", ExitCode.INVALID_ARGS) + } + + try { + // Validate the length and format + tool.migrationKeyHex = parseKey(migrationKeyHex) + } catch (KeyException e) { + if (tool.isVerbose) { + logger.error("Encountered an error", e) + } + tool.printUsageAndThrow(e.getMessage(), ExitCode.INVALID_ARGS) + } + } } String existingKeyHex = tool.migrationKeyHex ?: tool.keyHex - if (tool.handlingNiFiProperties) { + // Load NiFiProperties for either scenario; only encrypt if "handling" (see after flow XML) + if (tool.handlingNiFiProperties || tool.handlingFlowXml) { try { tool.niFiProperties = tool.loadNiFiProperties(existingKeyHex) } catch (Exception e) { tool.printUsageAndThrow("Cannot migrate key if no previous encryption occurred", ExitCode.ERROR_READING_NIFI_PROPERTIES) } - tool.niFiProperties = tool.encryptSensitiveProperties(tool.niFiProperties) } if (tool.handlingLoginIdentityProviders) { @@ -831,6 +1172,61 @@ class ConfigEncryptionTool { } tool.loginIdentityProviders = tool.encryptLoginIdentityProviders(tool.loginIdentityProviders) } + + if (tool.handlingFlowXml) { + try { + tool.flowXml = tool.loadFlowXml() + } catch (Exception e) { + tool.printUsageAndThrow("Cannot load flow.xml.gz", ExitCode.ERROR_READING_NIFI_PROPERTIES) + } + + // If the flow password was not set in nifi.properties, use the hard-coded default + String existingFlowPassword = tool.getExistingFlowPassword() + + // If the new password was not provided in the arguments, read from the console. If that is empty, use the same value (essentially a copy no-op) + String newFlowPassword = tool.flowPropertiesPassword ?: tool.getFlowPassword() + if (!newFlowPassword) { + newFlowPassword = existingFlowPassword + } + + // Get the algorithms and providers + NiFiProperties nfp = tool.niFiProperties + String existingAlgorithm = nfp?.getProperty(NiFiProperties.SENSITIVE_PROPS_ALGORITHM) ?: DEFAULT_FLOW_ALGORITHM + String existingProvider = nfp?.getProperty(NiFiProperties.SENSITIVE_PROPS_PROVIDER) ?: DEFAULT_PROVIDER + + String newAlgorithm = tool.newFlowAlgorithm ?: existingAlgorithm + String newProvider = tool.newFlowProvider ?: existingProvider + + tool.flowXml = tool.migrateFlowXmlContent(tool.flowXml, existingFlowPassword, newFlowPassword, existingAlgorithm, existingProvider, newAlgorithm, newProvider) + + // If the new key is the hard-coded internal value, don't persist it to nifi.properties + if (newFlowPassword != DEFAULT_NIFI_SENSITIVE_PROPS_KEY && newFlowPassword != existingFlowPassword) { + // Update the NiFiProperties object with the new flow password before it gets encrypted (wasteful, but NiFiProperties instances are immutable) + Properties rawProperties = new Properties() + nfp.getPropertyKeys().each { String k -> + rawProperties.put(k, nfp.getProperty(k)) + } + + // If the tool is not going to encrypt NiFiProperties and the existing file is already encrypted, encrypt and update the new sensitive props key + if (!tool.handlingNiFiProperties && existingNiFiPropertiesAreEncrypted) { + AESSensitivePropertyProvider spp = new AESSensitivePropertyProvider(tool.keyHex) + String encryptedSPK = spp.protect(newFlowPassword) + rawProperties.put(NiFiProperties.SENSITIVE_PROPS_KEY, encryptedSPK) + // Manually update the protection scheme or it will be lost + rawProperties.put(ProtectedNiFiProperties.getProtectionKey(NiFiProperties.SENSITIVE_PROPS_KEY), spp.getIdentifierKey()) + if (tool.isVerbose) { + logger.info("Tool is not configured to encrypt nifi.properties, but the existing nifi.properties is encrypted and flow.xml.gz was migrated, so manually persisting the new encrypted value to nifi.properties") + } + } else { + rawProperties.put(NiFiProperties.SENSITIVE_PROPS_KEY, newFlowPassword) + } + tool.niFiProperties = new StandardNiFiProperties(rawProperties) + } + } + + if (tool.handlingNiFiProperties) { + tool.niFiProperties = tool.encryptSensitiveProperties(tool.niFiProperties) + } } catch (CommandLineParseException e) { if (e.exitCode == ExitCode.HELP) { System.exit(ExitCode.HELP.ordinal()) @@ -846,8 +1242,13 @@ class ConfigEncryptionTool { try { // Do this as part of a transaction? synchronized (this) { - tool.writeKeyToBootstrapConf() - if (tool.handlingNiFiProperties) { + if (!tool.ignorePropertiesFiles) { + tool.writeKeyToBootstrapConf() + } + if (tool.handlingFlowXml) { + tool.writeFlowXmlToFile(tool.flowXml) + } + if (tool.handlingNiFiProperties || tool.handlingFlowXml) { tool.writeNiFiProperties() } if (tool.handlingLoginIdentityProviders) {
