NIFI-4942 [WIP] Added skeleton for secure hash handling in encrypt-config toolkit. Added test resource for Python scrypt implementation/verifier. Added unit tests.
NIFI-4942 [WIP] More unit tests passing. NIFI-4942 All unit tests pass and test artifacts are cleaned up. NIFI-4942 Added RAT exclusions. NIFI-4942 Added Scrypt hash format checker. Added unit tests. NIFI-4942 Added NiFi hash format checker. Added unit tests. NIFI-4942 Added check for simultaneous use of -z/-y. Added logic to check hashed password/key. Added logic to retrieve secure hash from file to compare. Added unit tests (125/125). NIFI-4942 Added new ExitCode. Added logic to return current hash params in JSON for Ambari to consume. Fixed typos in error messages. Added unit tests (129/129). NIFI-4942 Added Scrypt hash format verification for hash check. Added unit tests. NIFI-4942 Fixed RAT checks. Signed-off-by: Yolanda Davis <ymda...@apache.org> This closes #2628 Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/6d06defa Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/6d06defa Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/6d06defa Branch: refs/heads/master Commit: 6d06defa6336a0902180007182024219571e12e7 Parents: 82ac815 Author: Andy LoPresto <alopre...@apache.org> Authored: Thu Mar 22 20:47:53 2018 -0700 Committer: Yolanda Davis <ymda...@apache.org> Committed: Fri Apr 13 18:25:09 2018 -0400 ---------------------------------------------------------------------- .../security/util/crypto/scrypt/Scrypt.java | 43 +- .../util/scrypt/ScryptGroovyTest.groovy | 56 ++ .../nifi-toolkit-encrypt-config/pom.xml | 15 +- .../nifi/properties/ConfigEncryptionTool.groovy | 373 +++++++- .../properties/ConfigEncryptionToolTest.groovy | 919 ++++++++++++++++++- .../src/test/resources/scrypt.py | 18 + .../src/test/resources/secure_hash.key | 2 + .../src/test/resources/secure_hash_128.key | 2 + .../nifi/toolkit/tls/commandLine/ExitCode.java | 7 +- 9 files changed, 1398 insertions(+), 37 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/6d06defa/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java index 2aeae3d..b5622f8 100644 --- a/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java +++ b/nifi-commons/nifi-security-utils/src/main/java/org/apache/nifi/security/util/crypto/scrypt/Scrypt.java @@ -24,6 +24,7 @@ import java.security.GeneralSecurityException; import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; import org.apache.commons.codec.binary.Base64; @@ -51,6 +52,8 @@ public class Scrypt { private static final int DEFAULT_SALT_LENGTH = 16; + private static final Pattern SCRYPT_PATTERN = Pattern.compile("^\\$\\w{2}\\$\\w{5,}\\$[\\w\\/\\=\\+]{11,64}\\$[\\w\\/\\=\\+]{1,256}$"); + /** * Hash the supplied plaintext password and generate output in the format described * below: @@ -158,12 +161,12 @@ public class Scrypt { throw new IllegalArgumentException("Hash cannot be empty"); } - String[] parts = hashed.split("\\$"); - - if (parts.length != 5 || !parts[1].equals("s0")) { + if (!verifyHashFormat(hashed)) { throw new IllegalArgumentException("Hash is not properly formatted"); } + String[] parts = hashed.split("\\$"); + List<Integer> splitParams = parseParameters(parts[2]); int n = splitParams.get(0); int r = splitParams.get(1); @@ -189,6 +192,38 @@ public class Scrypt { } /** + * Returns true if the provided hash is a valid scrypt hash. Expected format: + * <p> + * {@code $s0$40801$ABCDEFGHIJKLMNOPQRSTUQ$hxU5g0eH6sRkBqcsiApI8jxvKRT+2QMCenV0GToiMQ8} + * <p> + * Components: + * <p> + * s0 -- version. Currently only "s0" is supported + * 40801 -- hex-encoded N, r, p parameters. {@see Scrypt#encodeParams()} for format + * ABCDEFGHIJKLMNOPQRSTUQ -- Base64-encoded (URL-safe, no padding) salt value. + * By default, 22 characters (16 bytes) but can be an arbitrary length between 11 and 64 characters (8 - 48 bytes) of random salt data + * hxU5g0eH6sRkBqcsiApI8jxvKRT+2QMCenV0GToiMQ8 -- the Base64-encoded (URL-safe, no padding) + * resulting hash component. By default, 43 characters (32 bytes) but can be an arbitrary length between 1 and MAX (depends on implementation, see RFC 7914) + * + * @param hash the hash to verify + * @return true if the format is acceptable + * @see Scrypt#formatSalt(byte[], int, int, int) + */ + public static boolean verifyHashFormat(String hash) { + if (StringUtils.isBlank(hash)) { + return false; + } + + // Currently, only version s0 is supported + if (!hash.startsWith("$s0$")) { + return false; + } + + // Check against the pattern + return SCRYPT_PATTERN.matcher(hash).matches(); + } + + /** * Parses the individual values from the encoded params value in the modified-mcrypt format for the salt & hash. * <p/> * Example: @@ -285,7 +320,7 @@ public class Scrypt { logger.warn("An empty salt was used for scrypt key derivation"); // throw new IllegalArgumentException("Salt cannot be empty"); // as the Exception is not being thrown, prevent NPE if salt is null by setting it to empty array - if( salt == null ) salt = new byte[]{}; + if (salt == null) salt = new byte[]{}; } if (saltLength < 8 || saltLength > 32) { http://git-wip-us.apache.org/repos/asf/nifi/blob/6d06defa/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/scrypt/ScryptGroovyTest.groovy ---------------------------------------------------------------------- diff --git a/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/scrypt/ScryptGroovyTest.groovy b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/scrypt/ScryptGroovyTest.groovy index 416d670..ad222a0 100644 --- a/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/scrypt/ScryptGroovyTest.groovy +++ b/nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/util/scrypt/ScryptGroovyTest.groovy @@ -397,4 +397,60 @@ class ScryptGroovyTest { assert msg =~ "Hash cannot be empty|Hash is not properly formatted" } } + + @Test + void testVerifyHashFormatShouldDetectValidHash() throws Exception { + // Arrange + final def VALID_HASHES = [ + "\$s0\$40801\$AAAAAAAAAAAAAAAAAAAAAA\$gLSh7ChbHdOIMvZ74XGjV6qF65d9qvQ8n75FeGnM8YM", + "\$s0\$40801\$ABCDEFGHIJKLMNOPQRSTUQ\$hxU5g0eH6sRkBqcsiApI8jxvKRT+2QMCenV0GToiMQ8", + "\$s0\$40801\$eO+UUcKYL2gnpD51QCc+gnywQ7Eg9tZeLMlf0XXr2zc\$99aTTB39TJo69aZCONQmRdyWOgYsDi+1MI+8D0EgMNM", + "\$s0\$40801\$AAAAAAAAAAAAAAAAAAAAAA\$Gk7K9YmlsWbd8FS7e4RKVWnkg9vlsqYnlD593pJ71gg", + "\$s0\$40801\$ABCDEFGHIJKLMNOPQRSTUQ\$Ri78VZbrp2cCVmGh2a9Nbfdov8LPnFb49MYyzPCaXmE", + "\$s0\$40801\$eO+UUcKYL2gnpD51QCc+gnywQ7Eg9tZeLMlf0XXr2zc\$rZIrP2qdIY7LN4CZAMgbCzl3YhXz6WhaNyXJXqFIjaI", + "\$s0\$40801\$AAAAAAAAAAAAAAAAAAAAAA\$GxH68bGykmPDZ6gaPIGOONOT2omlZ7cd0xlcZ9UsY/0", + "\$s0\$40801\$ABCDEFGHIJKLMNOPQRSTUQ\$KLGZjWlo59sbCbtmTg5b4k0Nu+biWZRRzhPhN7K5kkI", + "\$s0\$40801\$eO+UUcKYL2gnpD51QCc+gnywQ7Eg9tZeLMlf0XXr2zc\$6Ql6Efd2ac44ERoV31CL3Q0J3LffNZKN4elyMHux99Y", + // Uncommon but technically valid + "\$s0\$F0801\$AAAAAAAAAAA\$A", + "\$s0\$40801\$ABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOP\$A", + "\$s0\$40801\$ABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOP\$ABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOP", + "\$s0\$40801\$ABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOP\$ABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOPABCDEFGHIJKLMNOP", + "\$s0\$F0801\$AAAAAAAAAAA\$A", + "\$s0\$F0801\$AAAAAAAAAAA\$A", + "\$s0\$F0801\$AAAAAAAAAAA\$A", + "\$s0\$F0801\$AAAAAAAAAAA\$A", + "\$s0\$F0801\$AAAAAAAAAAA\$A", + ] + + // Act + VALID_HASHES.each { String validHash -> + logger.info("Using hash: ${validHash}") + + boolean isValidHash = Scrypt.verifyHashFormat(validHash) + logger.info("Hash is valid: ${isValidHash}") + + // Assert + assert isValidHash + } + } + + @Test + void testVerifyHashFormatShouldDetectInvalidHash() throws Exception { + // Arrange + + // Even though the spec allows for empty salts, the JCE does not, so extend enforcement of that to the user boundary + final def INVALID_HASHES = ['', null, '$s0$a0801$', '$s0$a0801$abcdefghijklmnopqrstuv$'] + + // Act + INVALID_HASHES.each { String invalidHash -> + logger.info("Using hash: ${invalidHash}") + + boolean isValidHash = Scrypt.verifyHashFormat(invalidHash) + logger.info("Hash is valid: ${isValidHash}") + + // Assert + assert !isValidHash + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/6d06defa/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml index 0bf1f74..2163de6 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/pom.xml @@ -13,7 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> <parent> <groupId>org.apache.nifi</groupId> <artifactId>nifi-toolkit</artifactId> @@ -162,7 +163,19 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <configuration> + <excludes combine.children="append"> + <exclude>src/test/resources/scrypt.py</exclude> + <exclude>src/test/resources/secure_hash.key</exclude> + <exclude>src/test/resources/secure_hash_128.key</exclude> + </excludes> + </configuration> + </plugin> </plugins> + </build> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/nifi/blob/6d06defa/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy ---------------------------------------------------------------------- diff --git a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy index 0e507c8..e8ac642 100644 --- a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy +++ b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy @@ -17,6 +17,7 @@ package org.apache.nifi.properties import groovy.io.GroovyPrintWriter +import groovy.json.JsonBuilder import groovy.xml.XmlUtil import org.apache.commons.cli.CommandLine import org.apache.commons.cli.CommandLineParser @@ -27,6 +28,8 @@ import org.apache.commons.cli.Options import org.apache.commons.cli.ParseException import org.apache.commons.codec.binary.Hex import org.apache.commons.io.IOUtils +import org.apache.nifi.security.util.crypto.CipherUtility +import org.apache.nifi.security.util.crypto.scrypt.Scrypt import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException import org.apache.nifi.toolkit.tls.commandLine.ExitCode import org.apache.nifi.util.NiFiProperties @@ -44,7 +47,9 @@ import javax.crypto.SecretKeyFactory import javax.crypto.spec.PBEKeySpec import javax.crypto.spec.PBEParameterSpec import java.nio.charset.StandardCharsets +import java.security.InvalidKeyException import java.security.KeyException +import java.security.MessageDigest import java.security.SecureRandom import java.security.Security import java.util.zip.GZIPInputStream @@ -62,11 +67,15 @@ class ConfigEncryptionTool { public String outputAuthorizersPath public String flowXmlPath public String outputFlowXmlPath + // If this value can be set by the running user, it can point to a manipulated file anywhere + private static String secureHashPath = "./secure_hash.key" private String keyHex private String migrationKeyHex private String password private String migrationPassword + private String secureHashKey + private String secureHashPassword // This is the raw value used in nifi.sensitive.props.key private String flowPropertiesPassword @@ -81,6 +90,7 @@ class ConfigEncryptionTool { private boolean usingPassword = true private boolean usingPasswordMigration = true + private boolean usingSecureHash = false private boolean migration = false private boolean isVerbose = false private boolean handlingNiFiProperties = false @@ -88,6 +98,7 @@ class ConfigEncryptionTool { private boolean handlingAuthorizers = false private boolean handlingFlowXml = false private boolean ignorePropertiesFiles = false + private boolean queryingCurrentHashParams = false private static final String HELP_ARG = "help" private static final String VERBOSE_ARG = "verbose" @@ -104,21 +115,30 @@ class ConfigEncryptionTool { private static final String PASSWORD_ARG = "password" private static final String KEY_MIGRATION_ARG = "oldKey" private static final String PASSWORD_MIGRATION_ARG = "oldPassword" + private static final String HASHED_KEY_MIGRATION_ARG = "secureHashKey" + private static final String HASHED_PASSWORD_MIGRATION_ARG = "secureHashPassword" private static final String USE_KEY_ARG = "useRawKey" private static final String MIGRATION_ARG = "migrate" private static final String PROPS_KEY_ARG = "propsKey" private static final String DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG = "encryptFlowXmlOnly" private static final String NEW_FLOW_ALGORITHM_ARG = "newFlowAlgorithm" private static final String NEW_FLOW_PROVIDER_ARG = "newFlowProvider" + private static final String CURRENT_HASH_PARAMS_ARG = "currentHashParams" + + // Static holder to avoid re-generating the options object multiple times in an invocation + private static Options staticOptions // Hard-coded fallback value from {@link org.apache.nifi.encrypt.StringEncryptor} private static final String DEFAULT_NIFI_SENSITIVE_PROPS_KEY = "nififtw!" private static final int MIN_PASSWORD_LENGTH = 12 - // Strong parameters as of 12 Aug 2016 + // Strong parameters as of 12 Aug 2016 (for key derivation) + // This value can remain an int until best practice specifies a value above 2**32 private static final int SCRYPT_N = 2**16 private static final int SCRYPT_R = 8 private static final int SCRYPT_P = 1 + static final String CURRENT_SCRYPT_VERSION = "s0" + private static final String NIFI_SCRYPT_PATTERN = /^\$\w{2}\$\w{5,}\$[\w\/\=\+]{22,}\$[\w\/\=\+]{43}$/ // Hard-coded values from StandardPBEByteEncryptor which will be removed during refactor of all flow encryption code in NIFI-1465 private static final int DEFAULT_KDF_ITERATIONS = 1000 @@ -142,7 +162,8 @@ class ConfigEncryptionTool { "files or in flow.xml.gz to be encrypted with a new key." private static final String LDAP_PROVIDER_CLASS = "org.apache.nifi.ldap.LdapProvider" - private static final String LDAP_PROVIDER_REGEX = /(?s)<provider>(?:(?!<provider>).)*?<class>\s*org\.apache\.nifi\.ldap\.LdapProvider.*?<\/provider>/ + private static + final String LDAP_PROVIDER_REGEX = /(?s)<provider>(?:(?!<provider>).)*?<class>\s*org\.apache\.nifi\.ldap\.LdapProvider.*?<\/provider>/ /* Explanation of LDAP_PROVIDER_REGEX: * (?s) -> single-line mode (i.e., `.` in regex matches newlines) * <provider> -> find occurrence of `<provider>` literally (case-sensitive) @@ -177,6 +198,7 @@ class ConfigEncryptionTool { private static final String DEFAULT_PROVIDER = BouncyCastleProvider.PROVIDER_NAME private static final String DEFAULT_FLOW_ALGORITHM = "PBEWITHMD5AND256BITAES-CBC-OPENSSL" + static private final int AMBARI_COMPATIBLE_SCRYPT_HASH_LENGTH = 256 private static String buildHeader(String description = DEFAULT_DESCRIPTION) { "${SEP}${description}${SEP * 2}" @@ -196,7 +218,11 @@ class ConfigEncryptionTool { ConfigEncryptionTool(String description) { this.header = buildHeader(description) - this.options = new Options() + this.options = getCliOptions() + } + + static Options buildOptions() { + Options options = new Options() options.addOption(Option.builder("h").longOpt(HELP_ARG).hasArg(false).desc("Show usage information (this message)").build()) options.addOption(Option.builder("v").longOpt(VERBOSE_ARG).hasArg(false).desc("Sets verbose mode (default false)").build()) options.addOption(Option.builder("n").longOpt(NIFI_PROPERTIES_ARG).hasArg(true).argName("file").desc("The nifi.properties file containing unprotected config values (will be overwritten unless -o is specified)").build()) @@ -212,23 +238,30 @@ class ConfigEncryptionTool { options.addOption(Option.builder("e").longOpt(KEY_MIGRATION_ARG).hasArg(true).argName("keyhex").desc("The old raw hexadecimal key to use during key migration").build()) options.addOption(Option.builder("p").longOpt(PASSWORD_ARG).hasArg(true).argName("password").desc("The password from which to derive the key to use to encrypt the sensitive properties").build()) options.addOption(Option.builder("w").longOpt(PASSWORD_MIGRATION_ARG).hasArg(true).argName("password").desc("The old password from which to derive the key during migration").build()) + options.addOption(Option.builder("y").longOpt(HASHED_KEY_MIGRATION_ARG).hasArg(true).argName("hashed_keyhex").desc("The old securely-hashed hexadecimal key to authenticate during key migration (see NiFi Admin Guide)").build()) + options.addOption(Option.builder("z").longOpt(HASHED_PASSWORD_MIGRATION_ARG).hasArg(true).argName("hashed_password").desc("The old securely-hashed password to authenticate during key migration (see NiFi Admin Guide)").build()) options.addOption(Option.builder("r").longOpt(USE_KEY_ARG).hasArg(false).desc("If provided, the secure console will prompt for the raw key value in hexadecimal form").build()) options.addOption(Option.builder("m").longOpt(MIGRATION_ARG).hasArg(false).desc("If provided, the nifi.properties and/or login-identity-providers.xml sensitive properties will be re-encrypted with a new key").build()) options.addOption(Option.builder("x").longOpt(DO_NOT_ENCRYPT_NIFI_PROPERTIES_ARG).hasArg(false).desc("If provided, the properties in flow.xml.gz will be re-encrypted with a new key but the nifi.properties and/or login-identity-providers.xml files will not be modified").build()) options.addOption(Option.builder("s").longOpt(PROPS_KEY_ARG).hasArg(true).argName("password|keyhex").desc("The password or key to use to encrypt the sensitive processor properties in flow.xml.gz").build()) options.addOption(Option.builder("A").longOpt(NEW_FLOW_ALGORITHM_ARG).hasArg(true).argName("algorithm").desc("The algorithm to use to encrypt the sensitive processor properties in flow.xml.gz").build()) options.addOption(Option.builder("P").longOpt(NEW_FLOW_PROVIDER_ARG).hasArg(true).argName("algorithm").desc("The security provider to use to encrypt the sensitive processor properties in flow.xml.gz").build()) + options.addOption(Option.builder().longOpt(CURRENT_HASH_PARAMS_ARG).hasArg(false).desc("Returns the current salt and cost params used to store the hashed key/password").build()) + options } static Options getCliOptions() { - return new ConfigEncryptionTool().options + if (!staticOptions) { + staticOptions = buildOptions() + } + return staticOptions } - /** - * Prints the usage message and available arguments for this tool (along with a specific error message if provided). - * - * @param errorMessage the optional error message - */ +/** + * Prints the usage message and available arguments for this tool (along with a specific error message if provided). + * + * @param errorMessage the optional error message + */ void printUsage(String errorMessage) { if (errorMessage) { System.out.println(errorMessage) @@ -236,7 +269,8 @@ class ConfigEncryptionTool { } HelpFormatter helpFormatter = new HelpFormatter() helpFormatter.setWidth(160) - helpFormatter.setOptionComparator(null) // preserve manual ordering of options when printing instead of alphabetical + helpFormatter.setOptionComparator(null) + // preserve manual ordering of options when printing instead of alphabetical helpFormatter.printHelp(ConfigEncryptionTool.class.getCanonicalName(), header, options, FOOTER, true) } @@ -257,6 +291,17 @@ class ConfigEncryptionTool { isVerbose = commandLine.hasOption(VERBOSE_ARG) + // If this flag is present, ensure no other options are present and then fail/return + if (commandLine.hasOption(CURRENT_HASH_PARAMS_ARG)) { + queryingCurrentHashParams = true + if (commandLineHasActionFlags(commandLine, [CURRENT_HASH_PARAMS_ARG])) { + printUsageAndThrow("When '--${CURRENT_HASH_PARAMS_ARG}' is specified, only '-h'/'--${HELP_ARG}' and '-v'/'--${VERBOSE_ARG}' are allowed", ExitCode.INVALID_ARGS) + } else { + // Otherwise return (avoid unnecessary parsing) + return commandLine + } + } + bootstrapConfPath = commandLine.getOptionValue(BOOTSTRAP_CONF_ARG) // If this flag is provided, the nifi.properties is necessary to read/write the flow encryption key, but the encryption process will not actually be applied to nifi.properties / login-identity-providers.xml @@ -361,6 +406,29 @@ class ConfigEncryptionTool { if (isVerbose) { logger.info("Key migration mode activated") } + if (isSecureHashArgumentPresent(commandLine)) { + logger.info("Secure hash argument present") + + // Check for old key/password and throw error + if (commandLine.hasOption(KEY_MIGRATION_ARG) || commandLine.hasOption(PASSWORD_MIGRATION_ARG)) { + printUsageAndThrow("If the '-w'/'--${PASSWORD_MIGRATION_ARG}' or '-e'/'--${KEY_MIGRATION_ARG}' arguments are present, '-z'/'--${HASHED_PASSWORD_MIGRATION_ARG}' and '-y'/'--${HASHED_KEY_MIGRATION_ARG}' cannot be used", ExitCode.INVALID_ARGS) + } + + // Check for both key and password and throw error + if (commandLine.hasOption(HASHED_KEY_MIGRATION_ARG) && commandLine.hasOption(HASHED_PASSWORD_MIGRATION_ARG)) { + printUsageAndThrow("Only one of '-z'/'--${HASHED_PASSWORD_MIGRATION_ARG}' and '-y'/'--${HASHED_KEY_MIGRATION_ARG}' can be used together", ExitCode.INVALID_ARGS) + } + + // Extract flags to field + if (commandLine.hasOption(HASHED_KEY_MIGRATION_ARG)) { + secureHashKey = commandLine.getOptionValue(HASHED_KEY_MIGRATION_ARG) + } else { + secureHashPassword = commandLine.getOptionValue(HASHED_PASSWORD_MIGRATION_ARG) + } + + // Set boolean flag to true + usingSecureHash = true + } if (commandLine.hasOption(PASSWORD_MIGRATION_ARG)) { usingPasswordMigration = true if (commandLine.hasOption(KEY_MIGRATION_ARG)) { @@ -370,7 +438,8 @@ class ConfigEncryptionTool { } } else { migrationKeyHex = commandLine.getOptionValue(KEY_MIGRATION_ARG) - usingPasswordMigration = !migrationKeyHex + // Use the "migration password" value if the migration key hex is absent and the secure hash password/key hex is absent (if either are present, the migration password is not) + usingPasswordMigration = !migrationKeyHex && !usingSecureHash } } else { if (commandLine.hasOption(PASSWORD_MIGRATION_ARG) || commandLine.hasOption(KEY_MIGRATION_ARG)) { @@ -410,15 +479,43 @@ class ConfigEncryptionTool { return commandLine } - /** - * The method returns the provided, derived, or securely-entered key in hex format. The reason the parameters must be provided instead of read from the fields is because this is used for the regular key/password and the migration key/password. - * - * @param device - * @param keyHex - * @param password - * @param usingPassword - * @return - */ + boolean commandLineHasActionFlags(CommandLine commandLine, List<String> acceptableOptionStrings = []) { + // Resolve the list of Option objects corresponding to "help" and "verbose" + final List<Option> ALWAYS_ACCEPTABLE_OPTIONS = resolveOptions([HELP_ARG, VERBOSE_ARG]) + + // Resolve the list of Option objects corresponding to the provided "additional acceptable options" + List<Option> acceptableOptions = resolveOptions(acceptableOptionStrings) + + // Determine the options submitted to the command line that are not acceptable + List<Option> invalidOptions = commandLine.options - (acceptableOptions + ALWAYS_ACCEPTABLE_OPTIONS) + if (invalidOptions) { + if (isVerbose) { + logger.error("In this mode, the following options are invalid: ${invalidOptions}") + } + return true + } else { + return false + } + } + + static List<Option> resolveOptions(List<String> strings) { + strings?.collect { String opt -> + getCliOptions().options.find { it.opt == opt || it.longOpt == opt } + } + } + + static boolean isSecureHashArgumentPresent(CommandLine commandLine) { + commandLine.hasOption(HASHED_PASSWORD_MIGRATION_ARG) || commandLine.hasOption(HASHED_KEY_MIGRATION_ARG) + } +/** + * The method returns the provided, derived, or securely-entered key in hex format. The reason the parameters must be provided instead of read from the fields is because this is used for the regular key/password and the migration key/password. + * + * @param device + * @param keyHex + * @param password + * @param usingPassword + * @return + */ private String getKeyInternal(TextDevice device = TextDevices.defaultTextDevice(), String keyHex, String password, boolean usingPassword) { if (usingPassword) { if (!password) { @@ -446,10 +543,89 @@ class ConfigEncryptionTool { } private String getMigrationKey() { - getKeyInternal(TextDevices.defaultTextDevice(), migrationKeyHex, migrationPassword, usingPasswordMigration) + if (usingSecureHash) { + // The boolean flag for "key" means the expression should evaluate to true when key is present and password is not + String knownHashValue = readSecureHashValueFromFile(secureHashKey && !secureHashPassword) + if (checkHashedValue(knownHashValue, getProvidedSecureHashValue())) { + // Retrieve the key from bootstrap.conf because the caller only has the hashed version available + return readMasterKeyFromBootstrap() + } else { + throw new InvalidKeyException("The provided hashed key/password is not correct") + } + } else { + return getKeyInternal(TextDevices.defaultTextDevice(), migrationKeyHex, migrationPassword, usingPasswordMigration) + } + } + + private String getProvidedSecureHashValue() { + if (usingSecureHash) { + return secureHashPassword ?: secureHashKey + } else { + return "" + } + } + + private static String readSecureHashValueFromFile(boolean readKey = true) { + File secureHashFile = new File(secureHashPath) + if (!secureHashFile.canRead()) { + throw new IOException("Cannot read from secure hash file") + } + List<String> lines = secureHashFile.readLines() + String linePrefix = readKey ? "secureHashKey" : "secureHashPassword" + String relevantLine = lines.find { it.startsWith(linePrefix) } + String hashValue = relevantLine?.split("=")?.last() + if (!hashValue) { + throw new InvalidKeyException("The secure hash of the ${readKey ? "key" : "password"} could not be read from the stored file") + } else { + return hashValue + } + } + + /** + * Returns true if the hash values are equivalent. Currently performs a *constant-time equality* check on the two values. As the scrypt format does not allow for reversing, two "equivalent" but non-identical hash values cannot be compared for equality. + * + * Example (byte equivalent): + * + * KHV: {@code $s0$40801$AAAAAAAAAAAAAAAAAAAAAA$gLSh7ChbHdOIMvZ74XGjV6qF65d9qvQ8n75FeGnM8YM} + * UHV: {@code $s0$40801$AAAAAAAAAAAAAAAAAAAAAA$gLSh7ChbHdOIMvZ74XGjV6qF65d9qvQ8n75FeGnM8YM} + * Result: EQUAL + * + * Example (semantically equivalent): + * + * KHV: {@code $s0$40801$AAAAAAAAAAAAAAAAAAAAAA$gLSh7ChbHdOIMvZ74XGjV6qF65d9qvQ8n75FeGnM8YM} + * UHV: {@code $s0$40801$ABCDEFGHIJKLMNOPQRSTUQ$hxU5g0eH6sRkBqcsiApI8jxvKRT+2QMCenV0GToiMQ8} + * Result: NOT EQUAL + * + * Even though both hash values are the result of the input "password", by design the hash values cannot be reversed to a common origin to determine their equality. If the raw input was known, both hashes could be determined to be valid, thus asserting the correctness of the raw input and the functional equivalence of the hash values. + * + * @param knownHashValue + * @param unknownHashValue + * @return true if the hash values are present and equal + */ + static boolean checkHashedValue(String knownHashValue, String unknownHashValue) { + if (!knownHashValue || !unknownHashValue) { + return false + } + + // The values should be in scrypt format + if (!verifyHashFormat(knownHashValue) || !verifyHashFormat(unknownHashValue)) { + return false + } + + return MessageDigest.isEqual(knownHashValue.getBytes(StandardCharsets.UTF_8), unknownHashValue.getBytes(StandardCharsets.UTF_8)) + } + + /** + * Returns true if the provided hash is in the valid format for NiFi secured hash storage. NiFi enforces additional constraints over the minimum Scrypt requirements (16+ byte [22+ B64] salt, 32 byte [43 B64] hash). + * + * @param hash the hash to verify + * @return true if the format is acceptable + */ + static boolean verifyHashFormat(String hash) { + hash =~ NIFI_SCRYPT_PATTERN } - private String getFlowPassword(TextDevice textDevice = TextDevices.defaultTextDevice()) { + private static String getFlowPassword(TextDevice textDevice = TextDevices.defaultTextDevice()) { readPasswordFromConsole(textDevice) } @@ -463,6 +639,10 @@ class ConfigEncryptionTool { new String(textDevice.readPassword()) } + private String readMasterKeyFromBootstrap() { + NiFiPropertiesLoader.extractKeyFromBootstrapFile(bootstrapConfPath) + } + /** * Returns the key in uppercase hexadecimal format with delimiters (spaces, '-', etc.) removed. All non-hex chars are removed. If the result is not a valid length (32, 48, 64 chars depending on the JCE), an exception is thrown. * @@ -831,7 +1011,9 @@ class ConfigEncryptionTool { try { def doc = new XmlSlurper().parseText(encryptedXml) // Find the provider element by class even if it has been renamed - def passwords = doc.userGroupProvider.find { it.'class' as String == LDAP_USER_GROUP_PROVIDER_CLASS }.property.findAll { + def passwords = doc.userGroupProvider.find { + it.'class' as String == LDAP_USER_GROUP_PROVIDER_CLASS + }.property.findAll { it.@name =~ "Password" && it.@encryption =~ "aes/gcm/\\d{3}" } @@ -1167,6 +1349,41 @@ class ConfigEncryptionTool { } } + /** + * Writes the contents of the secure hash configuration file (hashed value of current migration password/key hex) to the output {@code secure_hash.key} file. + * + * @throw IOException if there is a problem reading or writing the secure_hash.key file + */ + private void writeSecureHash() throws IOException { + if (!secureHashPath) { + throw new IllegalArgumentException("Cannot write hashed password/key to empty secure_hash.key path") + } + + File secureHashFile = new File(secureHashPath) + + if (isSafeToWrite(secureHashFile)) { + try { + List<String> secureHashFileLines = [] + // Calculate the secure hash of the current key (and password if provided) using current default values for cost params and random salt + String secureHashKey = secureHashKey(keyHex) + secureHashFileLines << "secureHashKey=${secureHashKey}" + if (password) { + String secureHashPassword = secureHashPassword(password) + secureHashFileLines << "secureHashPassword=${secureHashPassword}" + } + + // Write the updated values back to the file + secureHashFile.text = secureHashFileLines.join("\n") + } catch (IOException e) { + def msg = "Encountered an exception updating the secure_hash.key file with the hashed value(s)" + logger.error(msg, e) + throw e + } + } else { + throw new IOException("The secure_hash.key file at ${secureHashPath} must be writable by the user running this tool") + } + } + private static List<String> serializeNiFiPropertiesAndPreserveFormat(NiFiProperties niFiProperties, File originalPropertiesFile) { List<String> lines = originalPropertiesFile.readLines() @@ -1277,13 +1494,17 @@ class ConfigEncryptionTool { } // Generate a 128 bit salt - byte[] salt = generateScryptSalt() + byte[] salt = generateScryptSaltForKeyDerivation() int keyLengthInBytes = getValidKeyLengths().max() / 8 byte[] derivedKeyBytes = SCrypt.generate(password.getBytes(StandardCharsets.UTF_8), salt, SCRYPT_N, SCRYPT_R, SCRYPT_P, keyLengthInBytes) Hex.encodeHexString(derivedKeyBytes).toUpperCase() } - private static byte[] generateScryptSalt() { + /** + * Returns a static "raw" salt (the 128 bits of random data used when generating the hash, not the "complete" {@code $s0$e0101$ABCDEFGHIJKLMNOPQRSTUV} salt format). + * @return the raw salt in byte[] form + */ + private static byte[] generateScryptSaltForKeyDerivation() { // byte[] salt = new byte[16] // new SecureRandom().nextBytes(salt) // salt @@ -1294,6 +1515,32 @@ class ConfigEncryptionTool { "NIFI_SCRYPT_SALT".getBytes(StandardCharsets.UTF_8) } + private static byte[] generateRandomSalt(int bits = 128) { + byte[] salt = new byte[bits / 8] + new SecureRandom().nextBytes(salt) + salt + } + + /** + * Generates an scrypt salt using the provided parameters and encodes it in the proper format. If a value is not provided, the listed default will be used. + * + * @param rawSalt the raw 128 bit salt to use (default is to generate a random value) + * @param n the CPU/memory cost factor (default is {@value ConfigEncryptionTool#SCRYPT_N}) + * @param r the blocksize factor (default is {@value ConfigEncryptionTool#SCRYPT_R}) + * @param p the parallelization factor (default is {@value ConfigEncryptionTool#SCRYPT_P}) + * @param version the salt version identifier (default is {@value ConfigEncryptionTool#CURRENT_SCRYPT_VERSION}) + * @return the scrypt-formatted salt (e.g. {@code $s0$e0101$ABCDEFGHIJKLMNOPQRSTUV}) + */ + private + static String generateScryptSalt(byte[] rawSalt = generateRandomSalt(128), int n = SCRYPT_N, int r = SCRYPT_R, int p = SCRYPT_P, String version = CURRENT_SCRYPT_VERSION) { + String formattedSalt = Scrypt.formatSalt(rawSalt, n, r, p) + def versionString = "\$${version}\$" + if (!formattedSalt.startsWith(versionString)) { + formattedSalt = formattedSalt.replaceFirst(/$\w{2}$/, versionString) + } + return formattedSalt + } + private String getExistingFlowPassword() { return niFiProperties.getProperty(NiFiProperties.SENSITIVE_PROPS_KEY) as String ?: DEFAULT_NIFI_SENSITIVE_PROPS_KEY } @@ -1316,6 +1563,73 @@ class ConfigEncryptionTool { } } + static String secureHashPassword(String password, String salt = generateScryptSalt()) { + // If empty, generate complete salt + if (!salt) { + salt = generateScryptSalt() + } + + // The findAll() drops any empty elements + def saltComponents = salt.split('\\$').findAll() + if (saltComponents.size() < 3) { + throw new IllegalArgumentException("The provided salt was not in valid scrypt format") + } + if (saltComponents.first() != CURRENT_SCRYPT_VERSION) { + throw new IllegalArgumentException("Currently, only scrypt hashes with version ${CURRENT_SCRYPT_VERSION} are supported") + } + + // Split the encoded params into N, R, P + List<Integer> params = Scrypt.parseParameters(saltComponents[1]) + + // Generate the hashed format + Scrypt.scrypt(password, Base64.decoder.decode(saltComponents[2]), params[0], params[1], params[2], AMBARI_COMPATIBLE_SCRYPT_HASH_LENGTH) + } + + static String secureHashKey(String keyHex, String salt = generateScryptSalt()) { + // Lowercase the key hex, then call secureHashPassword() as the algorithm is the same + secureHashPassword(keyHex?.toLowerCase(), salt) + } + + + private String getCurrentHashParams() { + try { + int N + int r + int p + String base64Salt + + try { + String secureHash = readSecureHashValueFromFile() + + // The findAll() drops any empty elements + def hashComponents = secureHash.split('\\$').findAll() + if (hashComponents.size() < 3) { + throw new IllegalArgumentException("The provided secure hash was not in valid scrypt format") + } + if (hashComponents.first() != CURRENT_SCRYPT_VERSION) { + throw new IllegalArgumentException("Currently, only scrypt hashes with version ${CURRENT_SCRYPT_VERSION} are supported") + } + + // Split the encoded params into N, R, P + List<Integer> params = Scrypt.parseParameters(hashComponents[1]) + N = params[0] + r = params[1] + p = params[2] + base64Salt = hashComponents[2] + } catch (IOException | InvalidKeyException e) { + // These exceptions occur if the file doesn't exist, can't be read, or doesn't have secure hashes populated + N = SCRYPT_N + r = SCRYPT_R + p = SCRYPT_P + base64Salt = CipherUtility.encodeBase64NoPadding(generateRandomSalt()) + } + return new JsonBuilder([N: N, r: r, p: p, salt: base64Salt]).toString() + } catch (Exception e) { + logger.error("Encountered an exception getting current hash parameters: ${e.getLocalizedMessage()}") + printUsageAndThrow(e.getLocalizedMessage(), ExitCode.ERROR_READING_CONFIG) + } + } + /** * Runs main tool logic (parsing arguments, reading files, protecting properties, and writing key and properties out to destination files). * @@ -1330,6 +1644,12 @@ class ConfigEncryptionTool { try { tool.parse(args) + // Ensure the only content written to STDOUT is the JSON (consumable by other processes) + if (tool.queryingCurrentHashParams) { + System.out.println(tool.getCurrentHashParams()) + System.exit(ExitCode.SUCCESS.ordinal()) + } + boolean existingNiFiPropertiesAreEncrypted = tool.niFiPropertiesAreEncrypted() if (!tool.ignorePropertiesFiles || (tool.handlingFlowXml && existingNiFiPropertiesAreEncrypted)) { // If we are handling the flow.xml.gz and nifi.properties is already encrypted, try getting the key from bootstrap.conf rather than the console @@ -1471,6 +1791,9 @@ class ConfigEncryptionTool { synchronized (this) { if (!tool.ignorePropertiesFiles) { tool.writeKeyToBootstrapConf() + + // Always write the secure hash in case the next invocation needs it + tool.writeSecureHash() } if (tool.handlingFlowXml) { tool.writeFlowXmlToFile(tool.flowXml)
