This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new cc1e966 NIFI-8696: Added HashiCorp Vault KeyValue SPP
cc1e966 is described below
commit cc1e9665cda8ae91c2c067f567013efe69fe448c
Author: Joe Gresock <[email protected]>
AuthorDate: Tue Jul 27 21:30:02 2021 -0400
NIFI-8696: Added HashiCorp Vault KeyValue SPP
This closes #5255
Signed-off-by: David Handermann <[email protected]>
---
...actHashiCorpVaultSensitivePropertyProvider.java | 4 +-
...orpVaultKeyValueSensitivePropertyProvider.java} | 54 +++++-----
...iCorpVaultTransitSensitivePropertyProvider.java | 5 -
.../nifi/properties/PropertyProtectionScheme.java | 1 +
.../StandardSensitivePropertyProviderFactory.java | 2 +
.../HashiCorpVaultCommunicationService.java | 32 ++++--
...StandardHashiCorpVaultCommunicationService.java | 61 ++++++++++-
...andardHashiCorpVaultCommunicationServiceIT.java | 17 ++-
.../src/main/asciidoc/administration-guide.adoc | 117 ++++++++++++++++++++-
nifi-docs/src/main/asciidoc/toolkit-guide.adoc | 49 +--------
.../resources/conf/bootstrap-hashicorp-vault.conf | 3 +
.../resources/conf/bootstrap-hashicorp-vault.conf | 3 +
.../src/main/resources/conf/bootstrap.conf | 2 +-
13 files changed, 256 insertions(+), 94 deletions(-)
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
index 4570608..63c8c62 100644
---
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
@@ -116,7 +116,9 @@ public abstract class
AbstractHashiCorpVaultSensitivePropertyProvider extends Ab
* @param vaultBootstrapProperties The Vault-specific bootstrap properties
* @return true if the relevant Secrets Engine-specific properties are
configured
*/
- protected abstract boolean hasRequiredSecretsEngineProperties(final
BootstrapProperties vaultBootstrapProperties);
+ protected boolean hasRequiredSecretsEngineProperties(final
BootstrapProperties vaultBootstrapProperties) {
+ return getSecretsEnginePath(vaultBootstrapProperties) != null;
+ }
/**
* Returns the key used to identify the provider implementation in {@code
nifi.properties},
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultKeyValueSensitivePropertyProvider.java
similarity index 55%
copy from
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
copy to
nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultKeyValueSensitivePropertyProvider.java
index 6c7efd2..d373a5a 100644
---
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultKeyValueSensitivePropertyProvider.java
@@ -18,17 +18,16 @@ package org.apache.nifi.properties;
import org.apache.commons.lang3.StringUtils;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
+import java.util.Objects;
/**
- * Uses the HashiCorp Vault Transit Secrets Engine to encrypt sensitive values
at rest.
+ * Uses the HashiCorp Vault Key/Value (unversioned) Secrets Engine to store
sensitive values.
*/
-public class HashiCorpVaultTransitSensitivePropertyProvider extends
AbstractHashiCorpVaultSensitivePropertyProvider {
- private static final Charset PROPERTY_CHARSET = StandardCharsets.UTF_8;
- private static final String TRANSIT_PATH = "vault.transit.path";
+public class HashiCorpVaultKeyValueSensitivePropertyProvider extends
AbstractHashiCorpVaultSensitivePropertyProvider {
- HashiCorpVaultTransitSensitivePropertyProvider(final BootstrapProperties
bootstrapProperties) {
+ private static final String KEY_VALUE_PATH = "vault.kv.path";
+
+ HashiCorpVaultKeyValueSensitivePropertyProvider(final BootstrapProperties
bootstrapProperties) {
super(bootstrapProperties);
}
@@ -37,58 +36,55 @@ public class HashiCorpVaultTransitSensitivePropertyProvider
extends AbstractHash
if (vaultBootstrapProperties == null) {
return null;
}
- final String transitPath =
vaultBootstrapProperties.getProperty(TRANSIT_PATH);
+ final String kvPath =
vaultBootstrapProperties.getProperty(KEY_VALUE_PATH);
// Validate transit path
try {
-
PropertyProtectionScheme.fromIdentifier(getProtectionScheme().getIdentifier(transitPath));
+
PropertyProtectionScheme.fromIdentifier(getProtectionScheme().getIdentifier(kvPath));
} catch (IllegalArgumentException e) {
- throw new SensitivePropertyProtectionException(String.format("%s
[%s] contains unsupported characters", TRANSIT_PATH, transitPath), e);
+ throw new SensitivePropertyProtectionException(String.format("%s
[%s] contains unsupported characters", KEY_VALUE_PATH, kvPath), e);
}
- return transitPath;
+ return kvPath;
}
@Override
protected PropertyProtectionScheme getProtectionScheme() {
- return PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT;
- }
-
- @Override
- protected boolean hasRequiredSecretsEngineProperties(final
BootstrapProperties vaultBootstrapProperties) {
- return getSecretsEnginePath(vaultBootstrapProperties) != null;
+ return PropertyProtectionScheme.HASHICORP_VAULT_KV;
}
/**
- * Returns the encrypted cipher text.
+ * Stores the sensitive value in Vault and returns a description of the
secret.
*
* @param unprotectedValue the sensitive value
* @param context The property context, unused in this provider
* @return the value to persist in the {@code nifi.properties} file
- * @throws SensitivePropertyProtectionException if there is an exception
encrypting the value
+ * @throws SensitivePropertyProtectionException if there is an exception
writing the secret
*/
@Override
public String protect(final String unprotectedValue, final
ProtectedPropertyContext context) throws SensitivePropertyProtectionException {
if (StringUtils.isBlank(unprotectedValue)) {
- throw new IllegalArgumentException("Cannot encrypt an empty
value");
+ throw new IllegalArgumentException("Cannot protect an empty
value");
}
+ Objects.requireNonNull(context, "Context is required to protect a
value");
- return getVaultCommunicationService().encrypt(getPath(),
unprotectedValue.getBytes(PROPERTY_CHARSET));
+ getVaultCommunicationService().writeKeyValueSecret(getPath(),
context.getContextKey(), unprotectedValue);
+ return String.format("%s/%s", getPath(), context.getContextKey());
}
/**
- * Returns the decrypted plaintext.
+ * Returns the secret value, as read from Vault.
*
- * @param protectedValue the cipher text read from the {@code
nifi.properties} file
- * @param context The property context, unused in this provider
+ * @param protectedValue The value read from {@code nifi.properties} file.
Ignored in this provider.
+ * @param context The property context, from which the Vault secret name
is pulled
* @return the raw value to be used by the application
- * @throws SensitivePropertyProtectionException if there is an error
decrypting the cipher text
+ * @throws SensitivePropertyProtectionException if there is an error
retrieving the scret
*/
@Override
public String unprotect(final String protectedValue, final
ProtectedPropertyContext context) throws SensitivePropertyProtectionException {
- if (StringUtils.isBlank(protectedValue)) {
- throw new IllegalArgumentException("Cannot decrypt an empty
value");
- }
+ Objects.requireNonNull(context, "Context is required to unprotect a
value");
- return new String(getVaultCommunicationService().decrypt(getPath(),
protectedValue), PROPERTY_CHARSET);
+ return getVaultCommunicationService().readKeyValueSecret(getPath(),
context.getContextKey())
+ .orElseThrow(() -> new
SensitivePropertyProtectionException(String
+ .format("Secret [%s] not found in Vault Key/Value
engine at [%s]", context.getContextKey(), getPath())));
}
}
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
index 6c7efd2..452ee14 100644
---
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
@@ -53,11 +53,6 @@ public class HashiCorpVaultTransitSensitivePropertyProvider
extends AbstractHash
return PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT;
}
- @Override
- protected boolean hasRequiredSecretsEngineProperties(final
BootstrapProperties vaultBootstrapProperties) {
- return getSecretsEnginePath(vaultBootstrapProperties) != null;
- }
-
/**
* Returns the encrypted cipher text.
*
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
index 0e2a72c..ee3859b 100644
---
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
@@ -26,6 +26,7 @@ import java.util.Objects;
public enum PropertyProtectionScheme {
AES_GCM("aes/gcm/(128|192|256)", "aes/gcm/%s", "AES Sensitive Property
Provider", true),
AWS_KMS("aws/kms", "aws/kms", "AWS KMS Sensitive Property Provider",
false),
+ HASHICORP_VAULT_KV("hashicorp/vault/kv/[a-zA-Z0-9_-]+",
"hashicorp/vault/kv/%s", "HashiCorp Vault Key/Value Engine Sensitive Property
Provider", false),
HASHICORP_VAULT_TRANSIT("hashicorp/vault/transit/[a-zA-Z0-9_-]+",
"hashicorp/vault/transit/%s", "HashiCorp Vault Transit Engine Sensitive
Property Provider", false);
PropertyProtectionScheme(final String identifierPattern, final String
identifierFormat, final String name, final boolean requiresSecretKey) {
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
index cfbb90d..9ba0178 100644
---
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
@@ -124,6 +124,8 @@ public class StandardSensitivePropertyProviderFactory
implements SensitiveProper
return providerMap.computeIfAbsent(protectionScheme, s -> new
AWSSensitivePropertyProvider(getBootstrapProperties()));
case HASHICORP_VAULT_TRANSIT:
return providerMap.computeIfAbsent(protectionScheme, s -> new
HashiCorpVaultTransitSensitivePropertyProvider(getBootstrapProperties()));
+ case HASHICORP_VAULT_KV:
+ return providerMap.computeIfAbsent(protectionScheme, s -> new
HashiCorpVaultKeyValueSensitivePropertyProvider(getBootstrapProperties()));
default:
throw new SensitivePropertyProtectionException("Unsupported
protection scheme " + protectionScheme);
}
diff --git
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java
index 977b369..bf43268 100644
---
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java
+++
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultCommunicationService.java
@@ -16,6 +16,8 @@
*/
package org.apache.nifi.vault.hashicorp;
+import java.util.Optional;
+
/**
* A service to handle all communication with an instance of HashiCorp Vault.
* @see <a href="https://www.vaultproject.io/";>https://www.vaultproject.io/</a>
@@ -26,21 +28,39 @@ public interface HashiCorpVaultCommunicationService {
* Encrypts the given plaintext using Vault's Transit Secrets Engine.
*
* @see <a
href="https://www.vaultproject.io/api-docs/secret/transit";>https://www.vaultproject.io/api-docs/secret/transit</a>
- * @param transitKey A named encryption key used in the Transit Secrets
Engine. The key is expected to have
- * already been configured in the Vault instance.
+ * @param transitPath The Vault path to use for the configured Transit
Secrets Engine
* @param plainText The plaintext to encrypt
* @return The cipher text
*/
- String encrypt(String transitKey, byte[] plainText);
+ String encrypt(String transitPath, byte[] plainText);
/**
* Decrypts the given cipher text using Vault's Transit Secrets Engine.
*
* @see <a
href="https://www.vaultproject.io/api-docs/secret/transit";>https://www.vaultproject.io/api-docs/secret/transit</a>
- * @param transitKey A named encryption key used in the Transit Secrets
Engine. The key is expected to have
- * already been configured in the Vault instance.
+ * @param transitPath The Vault path to use for the configured Transit
Secrets Engine
* @param cipherText The cipher text to decrypt
* @return The decrypted plaintext
*/
- byte[] decrypt(String transitKey, String cipherText);
+ byte[] decrypt(String transitPath, String cipherText);
+
+ /**
+ * Writes a secret using Vault's unversioned Key/Value Secrets Engine.
+ *
+ * @see <a
href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1";>https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
+ * @param keyValuePath The Vault path to use for the configured Key/Value
v1 Secrets Engine
+ * @param key The secret key
+ * @param value The secret value
+ */
+ void writeKeyValueSecret(String keyValuePath, String key, String value);
+
+ /**
+ * Reads a secret from Vault's unversioned Key/Value Secrets Engine.
+ *
+ * @see <a
href="https://www.vaultproject.io/api-docs/secret/kv/kv-v1";>https://www.vaultproject.io/api-docs/secret/kv/kv-v1</a>
+ * @param keyValuePath The Vault path to use for the configured Key/Value
v1 Secrets Engine
+ * @param key The secret key
+ * @return The secret value, or empty if not found
+ */
+ Optional<String> readKeyValueSecret(String keyValuePath, String key);
}
diff --git
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
index 61f0176..21c9213 100644
---
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
+++
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
@@ -16,25 +16,35 @@
*/
package org.apache.nifi.vault.hashicorp;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperties;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultPropertySource;
import org.springframework.core.env.PropertySource;
import org.springframework.vault.authentication.SimpleSessionManager;
import org.springframework.vault.client.ClientHttpRequestFactoryFactory;
+import org.springframework.vault.core.VaultKeyValueOperations;
import org.springframework.vault.core.VaultTemplate;
import org.springframework.vault.core.VaultTransitOperations;
import org.springframework.vault.support.Ciphertext;
import org.springframework.vault.support.Plaintext;
+import org.springframework.vault.support.VaultResponseSupport;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import static
org.springframework.vault.core.VaultKeyValueOperationsSupport.KeyValueBackend.KV_1;
/**
* Implements the VaultCommunicationService using Spring Vault
*/
public class StandardHashiCorpVaultCommunicationService implements
HashiCorpVaultCommunicationService {
-
private final HashiCorpVaultConfiguration vaultConfiguration;
private final VaultTemplate vaultTemplate;
private final VaultTransitOperations transitOperations;
+ private final Map<String, VaultKeyValueOperations> keyValueOperationsMap;
/**
* Creates a VaultCommunicationService that uses Spring Vault.
@@ -49,6 +59,7 @@ public class StandardHashiCorpVaultCommunicationService
implements HashiCorpVaul
new
SimpleSessionManager(vaultConfiguration.clientAuthentication()));
transitOperations = vaultTemplate.opsForTransit();
+ keyValueOperationsMap = new HashMap<>();
}
/**
@@ -61,12 +72,52 @@ public class StandardHashiCorpVaultCommunicationService
implements HashiCorpVaul
}
@Override
- public String encrypt(final String transitKey, final byte[] plainText) {
- return transitOperations.encrypt(transitKey,
Plaintext.of(plainText)).getCiphertext();
+ public String encrypt(final String transitPath, final byte[] plainText) {
+ return transitOperations.encrypt(transitPath,
Plaintext.of(plainText)).getCiphertext();
}
@Override
- public byte[] decrypt(final String transitKey, final String cipherText) {
- return transitOperations.decrypt(transitKey,
Ciphertext.of(cipherText)).getPlaintext();
+ public byte[] decrypt(final String transitPath, final String cipherText) {
+ return transitOperations.decrypt(transitPath,
Ciphertext.of(cipherText)).getPlaintext();
+ }
+
+ /**
+ * Writes the value to the "value" key of the secret with the path
[keyValuePath]/[key].
+ * @param keyValuePath The Vault path to use for the configured Key/Value
v1 Secrets Engine
+ * @param key The secret key
+ * @param value The secret value
+ */
+ @Override
+ public void writeKeyValueSecret(final String keyValuePath, final String
key, final String value) {
+ final VaultKeyValueOperations keyValueOperations =
keyValueOperationsMap
+ .computeIfAbsent(keyValuePath, path ->
vaultTemplate.opsForKeyValue(path, KV_1));
+ keyValueOperations.put(key, new SecretData(value));
+ }
+
+ /**
+ * Returns the value of the "value" key from the secret at the path
[keyValuePath]/[key].
+ * @param keyValuePath The Vault path to use for the configured Key/Value
v1 Secrets Engine
+ * @param key The secret key
+ * @return The value of the secret
+ */
+ @Override
+ public Optional<String> readKeyValueSecret(final String keyValuePath,
final String key) {
+ final VaultKeyValueOperations keyValueOperations =
keyValueOperationsMap
+ .computeIfAbsent(keyValuePath, path ->
vaultTemplate.opsForKeyValue(path, KV_1));
+ final VaultResponseSupport<SecretData> response =
keyValueOperations.get(key, SecretData.class);
+ return response == null ? Optional.empty() :
Optional.ofNullable(response.getRequiredData().getValue());
+ }
+
+ private static class SecretData {
+ private final String value;
+
+ @JsonCreator
+ public SecretData(@JsonProperty("value") final String value) {
+ this.value = value;
+ }
+
+ public String getValue() {
+ return value;
+ }
}
}
diff --git
a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationServiceIT.java
b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationServiceIT.java
index 60d64a9..f347801 100644
---
a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationServiceIT.java
+++
b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationServiceIT.java
@@ -17,17 +17,19 @@
package org.apache.nifi.vault.hashicorp;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperties;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import java.nio.charset.StandardCharsets;
+import static org.junit.Assert.assertEquals;
+
/**
* The simplest way to run this test is by installing Vault locally, then
running:
*
* vault server -dev
* vault secrets enable transit
+ * vault secrets enable kv
* vault write -f transit/keys/nifi
*
* Make note of the Root Token and create a properties file with the contents:
@@ -62,6 +64,17 @@ public class StandardHashiCorpVaultCommunicationServiceIT {
byte[] decrypted = vcs.decrypt(TRANSIT_KEY, ciphertext);
- Assert.assertEquals(plaintext, new String(decrypted,
StandardCharsets.UTF_8));
+ assertEquals(plaintext, new String(decrypted, StandardCharsets.UTF_8));
+ }
+
+ @Test
+ public void testReadWriteSecret() {
+ final String key = "key";
+ final String value = "value";
+
+ vcs.writeKeyValueSecret("kv", key, value);
+
+ final String resultValue = vcs.readKeyValueSecret("kv",
key).orElseThrow(() -> new NullPointerException("Missing secret for kv/key"));
+ assertEquals(value, resultValue);
}
}
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc
b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index 309ead9..4f71047 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -1765,7 +1765,7 @@ All options require a password
(`nifi.sensitive.props.key` value) of *at least 1
[[encrypt-config_tool]]
== Encrypted Passwords in Configuration Files
-In order to facilitate the secure setup of NiFi, you can use the
`encrypt-config` command line utility to encrypt raw configuration values that
NiFi decrypts in memory on startup. This extensible protection scheme
transparently allows NiFi to use raw values in operation, while protecting them
at rest. In addition to the default AES encryption provider, a HashiCorp Vault
encryption provider can be configured in the
`bootstrap-hashicorp-vault.properties` file.
+In order to facilitate the secure setup of NiFi, you can use the
`encrypt-config` command line utility to encrypt raw configuration values that
NiFi decrypts in memory on startup. This extensible protection scheme
transparently allows NiFi to use raw values in operation, while protecting them
at rest.
This is a change in behavior; prior to 1.0, all configuration values were
stored in plaintext on the file system. POSIX file permissions were recommended
to limit unauthorized access to these files.
@@ -1773,6 +1773,121 @@ If no administrator action is taken, the configuration
values remain unencrypted
For more information, see the
<<toolkit-guide.adoc#encrypt_config_tool,Encrypt-Config Tool>> section in the
link:toolkit-guide.html[NiFi Toolkit Guide].
+In addition to the default AES encryption provider, other providers can be
configured in their respective `bootstrap-*.conf` files. Following is a list of
additional encryption providers and their configuration:
+
+=== HashiCorp Vault providers
+Two encryption providers are currently configurable in the
`bootstrap-hashicorp-vault.conf` file:
+
+[options="header,footer"]
+|===
+|Provider|Provider Identifier|Description
+|HashiCorp Vault Transit
provider|`hashicorp/vault/kv/{vault.transit.path}`|Uses HashiCorp Vault's
Transit Secrets Engine to decrypt sensitive properties.
+|HashiCorp Vault Key/Value
provider|`hashicorp/vault/kv/{vault.transit.path}`|Retrieves sensitive values
from Secrets stored in a HashiCorp Vault Key/Value (unversioned) Secrets Engine.
+|===
+
+Note that all HashiCorp Vault encryption providers require a running Vault
instance in order to decrypt these values at NiFi's startup.
+
+Following are the configuration properties available inside the
`bootstrap-hashicorp-vault.conf` file:
+
+==== Required properties
+
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`vault.uri`|The HashiCorp Vault URI (e.g., `https://vault-server:8200`). If
not set, all HashiCorp Vault providers will be disabled.|_none_
+|`vault.authentication.properties.file`|Filename of a properties file
containing Vault authentication properties. See the `Authentication-specific
property keys` section of
https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration
for all authentication property keys. If not set, all Spring Vault
authentication properties must be configured directly in
bootstrap-hashicorp-vault.conf.|_none_
+|`vault.transit.path`|If set, enables the HashiCorp Vault Transit provider.
The value should be the Vault `path` of a Transit Secrets Engine (e.g.,
`nifi-transit`). Valid characters include alphanumeric, dash, and
underscore.|_none_
+|`vault.kv.path`|If set, enables the HashiCorp Vault Key/Value provider. The
value should be the Vault `path` of a K/V (v1) Secrets Engine (e.g.,
`nifi-kv`). Valid characters include alphanumeric, dash, and underscore.|_none_
+|===
+
+==== Optional properties
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`vault.connection.timeout`|The connection timeout of the Vault client|`5 secs`
+|`vault.read.timeout`|The read timeout of the Vault client|`15 secs`
+|`vault.ssl.enabledCipherSuites`|A comma-separated list of the enabled TLS
cipher suites|_none_
+|`vault.ssl.enabledProtocols`|A comma-separated list of the enabled TLS
protocols|_none_
+|`vault.ssl.key-store`|Path to a keystore. Required if the Vault server is
TLS-enabled|_none_
+|`vault.ssl.key-store-type`|Keystore type (JKS, BCFKS or PKCS12). Required if
the Vault server is TLS-enabled|_none_
+|`vault.ssl.key-store-password`|Keystore password. Required if the Vault
server is TLS-enabled|_none_
+|`vault.ssl.trust-store`|Path to a truststore. Required if the Vault server
is TLS-enabled|_none_
+|`vault.ssl.trust-store-type`|Truststore type (JKS, BCFKS or PKCS12).
Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.trust-store-password`|Truststore password. Required if the Vault
server is TLS-enabled|_none_
+|===
+
+=== AWS KMS provider
+This provider uses AWS Key Management Service (https://aws.amazon.com/kms/)
for decryption. AWS KMS configuration properties can be stored in the
`bootstrap-aws.conf` file, as referenced in `bootstrap.conf`. If the
configuration properties are not specified in `bootstrap-aws.conf`, then the
provider will attempt to use the AWS default credentials provider, which checks
standard environment variables and system properties.
+
+==== Required properties
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`aws.kms.key.id`|The identifier or ARN that the AWS KMS client uses for
encryption and decryption.|_none_
+|===
+
+==== Optional properties
+===== All of the following must be configured, or will be ignored entirely.
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`aws.region`|The AWS region used to configure the AWS KMS Client.|_none_
+|`aws.access.key.id`|The access key ID credential used to access AWS
KMS.|_none_
+|`aws.secret.access.key`|The secret access key used to access AWS KMS.|_none_
+|===
+
+=== Property Context Mapping
+Some encryption providers store protected values in an external service
instead of persisting the encrypted values directly in the configuration file.
To support this use case, a property context is defined for each protected
property in NiFi's configuration files, in the format:
`{context-name}/{property-name}`
+
+* `context-name` - represents a namespace for properties in order to
disambiguate properties with the same name. Without additional configuration,
all protected properties are assigned the `default` context.
+* `property-name` - contains the name of the property.
+
+In order to support logical context names, mapping properties may be provided
in `bootstrap.conf`, as follows:
+
+```
+nifi.bootstrap.protection.context.mapping.<context-name>=<identifier matching
regex>
+```
+
+Here, `context-name` would determine the context name above, and `<identifier
matching regex>` would map any property whose *group identifier* matched the
provided Regular Expression. *Group identifiers* are defined per configuration
file type, and are described as follows:
+[options="header,footer"]
+|===
+|Configuration File|Group Identifier Description|Assigned Context
+|`nifi.properties`|There is no concept of a group identifier here, since all
property names should be unique.|_default_
+|`authorizers.xml`|The `<identifier>` value of the XML block surrounding the
property.|The mapped context name if RegEx matches the identifier, otherwise
_default_
+|`login-identity-providers.xml`|The `<identifier>` value of the XML block
surrounding the property.|The mapped context name if RegEx matches the
identifier, otherwise _default_
+|===
+
+==== Example
+In the NiFi binary distribution, the `login-identity-providers.xml` file comes
with a provider with the identifier `ldap-provider` and a property called
`Manager Password`:
+
+```
+ <provider>
+ <identifier>ldap-provider</identifier>
+ <class>org.apache.nifi.ldap.LdapProvider</class>
+ ...
+ <property name="Manager Password"/>
+ ...
+ </provider>
+```
+Similarly, the `authorizers.xml` file comes with a `ldap-user-group-provider`
and a property also called `Manager Password`:
+
+```
+ <userGroupProvider>
+ <identifier>ldap-user-group-provider</identifier>
+ <class>org.apache.nifi.ldap.tenants.LdapUserGroupProvider</class>
+ ...
+ <property name="Manager Password"/>
+ ...
+ </userGroupProvider>
+```
+
+If the Manager Password is desired to reference the same exact property (e.g.,
the same Secret in the HashiCorp Vault K/V provider) but still be distinguished
from any other `Manager Password` property unrelated to LDAP, the following
mapping could be added:
+
+```
+nifi.bootstrap.protection.context.mapping.ldap=ldap-.*
+```
+
+This would cause both of the above to be assigned a context of `"ldap/Manager
Password"` instead of `"default/Manager Password"`.
[[admin-toolkit]]
== NiFi Toolkit Administrative Tools
In addition to `tls-toolkit` and `encrypt-config`, the NiFi Toolkit also
contains command line utilities for administrators to support NiFi maintenance
in standalone and clustered environments. These utilities include:
diff --git a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
index f61780b..3b173a8 100644
--- a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
@@ -477,52 +477,13 @@ The protection scheme can be selected during encryption
using the `--protectionS
The default protection scheme, `AES-G/CM` simply encrypts sensitive properties
and marks their protection as either `aes/gcm/256` or `aes/gcm/256` as
appropriate. This protection is all done within NiFi itself.
==== HASHICORP_VAULT_TRANSIT
-This protection scheme uses HashiCorp Vault's Transit Secrets Engine
(https://www.vaultproject.io/docs/secrets/transit) to outsource encryption to a
configured Vault server. All HashiCorp Vault configuration is stored in the
`bootstrap-hashicorp-vault.conf` file, as referenced in the `bootstrap.conf` of
a NiFi or NiFi Registry instance. Therefore, when using the
HASHICORP_VAULT_TRANSIT protection scheme, the
`nifi(.registry)?.bootstrap.protection.hashicorp.vault.conf` property in the `b
[...]
-
-===== Required properties
-[options="header,footer"]
-|===
-|Property Name|Description|Default
-|`vault.uri`|The HashiCorp Vault URI (e.g., `https://vault-server:8200`). If
not set, this provider will be disabled.|_none_
-|`vault.authentication.properties.file`|Filename of a properties file
containing Vault authentication properties. See the `Authentication-specific
property keys` section of
https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration
for all authentication property keys. If not set, all Spring Vault
authentication properties must be configured directly in
bootstrap-hashicorp-vault.conf.|_none_
-|`vault.transit.path`|The HashiCorp Vault `path` specifying the Transit
Secrets Engine (e.g., `nifi-transit`). Valid characters include alphanumeric,
dash, and underscore. If not set, this provider will be disabled.|_none_
-|===
-
-===== Optional properties
-[options="header,footer"]
-|===
-|Property Name|Description|Default
-|`vault.connection.timeout`|The connection timeout of the Vault client|`5 secs`
-|`vault.read.timeout`|The read timeout of the Vault client|`15 secs`
-|`vault.ssl.enabledCipherSuites`|A comma-separated list of the enabled TLS
cipher suites|_none_
-|`vault.ssl.enabledProtocols`|A comma-separated list of the enabled TLS
protocols|_none_
-|`vault.ssl.key-store`|Path to a keystore. Required if the Vault server is
TLS-enabled|_none_
-|`vault.ssl.key-store-type`|Keystore type (JKS, BCFKS or PKCS12). Required if
the Vault server is TLS-enabled|_none_
-|`vault.ssl.key-store-password`|Keystore password. Required if the Vault
server is TLS-enabled|_none_
-|`vault.ssl.trust-store`|Path to a truststore. Required if the Vault server
is TLS-enabled|_none_
-|`vault.ssl.trust-store-type`|Truststore type (JKS, BCFKS or PKCS12).
Required if the Vault server is TLS-enabled|_none_
-|`vault.ssl.trust-store-password`|Truststore password. Required if the Vault
server is TLS-enabled|_none_
-|===
+This protection scheme uses HashiCorp Vault's Transit Secrets Engine
(https://www.vaultproject.io/docs/secrets/transit) to outsource encryption to a
configured Vault server. All HashiCorp Vault configuration is stored in the
`bootstrap-hashicorp-vault.conf` file, as referenced in the `bootstrap.conf` of
a NiFi or NiFi Registry instance. Therefore, when using the
HASHICORP_VAULT_TRANSIT protection scheme, the
`nifi(.registry)?.bootstrap.protection.hashicorp.vault.conf` property in the `b
[...]
+
+==== HASHICORP_VAULT_KV
+This protection scheme uses HashiCorp Vault's Transit unversioned Key/Value
Engine (https://www.vaultproject.io/docs/secrets/kv/kv-v1) to store sensitive
values as Vault Secrets. All HashiCorp Vault configuration is stored in the
`bootstrap-hashicorp-vault.conf` file, as referenced in the `bootstrap.conf` of
a NiFi or NiFi Registry instance. Therefore, when using the HASHICORP_VAULT_KV
protection scheme, the
`nifi(.registry)?.bootstrap.protection.hashicorp.vault.conf` property in the `b
[...]
==== AWS_KMS
-This protection scheme uses AWS Key Management Service
(https://aws.amazon.com/kms/) for encryption and decryption. AWS KMS
configuration properties can be stored in the `bootstrap-aws.conf` file, as
referenced in the `bootstrap.conf` of NiFi or NiFi Registry. If the
configuration properties are not specified in `bootstrap-aws.conf`, then the
provider will attempt to use the AWS default credentials provider, which checks
standard environment variables and system properties.
-
-===== Required properties
-[options="header,footer"]
-|===
-|Property Name|Description|Default
-|`aws.kms.key.id`|The identifier or ARN that the AWS KMS client uses for
encryption and decryption.|_none_
-|===
-
-===== Optional properties
-====== All of the following must be configured, or will be ignored entirely.
-[options="header,footer"]
-|===
-|Property Name|Description|Default
-|`aws.region`|The AWS region used to configure the AWS KMS Client.|_none_
-|`aws.access.key.id`|The access key ID credential used to access AWS
KMS.|_none_
-|`aws.secret.access.key`|The secret access key used to access AWS KMS.|_none_
-|===
+This protection scheme uses AWS Key Management Service
(https://aws.amazon.com/kms/) for encryption and decryption. AWS KMS
configuration properties can be stored in the `bootstrap-aws.conf` file, as
referenced in the `bootstrap.conf` of NiFi or NiFi Registry. If the
configuration properties are not specified in `bootstrap-aws.conf`, then the
provider will attempt to use the AWS default credentials provider, which checks
standard environment variables and system properties. Therefore, w [...]
=== Examples
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
index 1d1a409..bbf54f5 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
@@ -21,6 +21,9 @@ vault.uri=
# Transit Path is required to enable the Sensitive Properties Provider
Protection Scheme 'hashicorp/vault/transit/{path}'
vault.transit.path=
+# Key/Value Path is required to enable the Sensitive Properties Provider
Protection Scheme 'hashicorp/vault/kv/{path}'
+vault.kv.path=
+
# Token Authentication example properties
# vault.authentication=TOKEN
# vault.token=<token value>
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
index 1d1a409..bbf54f5 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
+++
b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
@@ -21,6 +21,9 @@ vault.uri=
# Transit Path is required to enable the Sensitive Properties Provider
Protection Scheme 'hashicorp/vault/transit/{path}'
vault.transit.path=
+# Key/Value Path is required to enable the Sensitive Properties Provider
Protection Scheme 'hashicorp/vault/kv/{path}'
+vault.kv.path=
+
# Token Authentication example properties
# vault.authentication=TOKEN
# vault.token=<token value>
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
index 31e397c..046d252 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
+++
b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
@@ -59,4 +59,4 @@ nifi.registry.bootstrap.sensitive.key=
nifi.registry.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
# AWS KMS Sensitive Property Providers
-nifi.registry.bootstrap.protection.aws.kms.conf=./conf/bootstrap-aws.conf
\ No newline at end of file
+nifi.registry.bootstrap.protection.aws.kms.conf=./conf/bootstrap-aws.conf
