This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 726082f NIFI-8447 Added HashiCorp Vault Transit Sensitive Properties
Provider
726082f is described below
commit 726082ffa6c9f4b350fd6026152c50dc5bae2151
Author: Joe Gresock <[email protected]>
AuthorDate: Mon Jun 14 06:53:37 2021 -0400
NIFI-8447 Added HashiCorp Vault Transit Sensitive Properties Provider
- Added default bootstrap-hashicorp-vault.conf
- Updated Toolkit Guide documentation with HashiCorp Vault properties
This closes #5154
Signed-off-by: David Handermann <[email protected]>
---
.../AbstractBootstrapPropertiesLoader.java | 33 ++++--
.../nifi/properties/BootstrapProperties.java | 75 +++++++++++-
.../nifi-sensitive-property-provider/pom.xml | 11 ++
.../properties/AESSensitivePropertyProvider.java | 2 +-
...actHashiCorpVaultSensitivePropertyProvider.java | 132 +++++++++++++++++++++
.../AbstractSensitivePropertyProvider.java | 13 --
...iCorpVaultTransitSensitivePropertyProvider.java | 92 ++++++++++++++
.../nifi/properties/PropertyProtectionScheme.java | 3 +-
.../StandardSensitivePropertyProviderFactory.java | 10 +-
...andardSensitivePropertyProviderFactoryTest.java | 85 +++++++++++--
...StandardHashiCorpVaultCommunicationService.java | 48 ++------
.../config/HashiCorpVaultConfiguration.java | 105 +++++++++++++++-
.../hashicorp/config/HashiCorpVaultProperties.java | 1 +
.../hashicorp/config/HashiCorpVaultProperty.java | 1 +
.../config/HashiCorpVaultPropertySource.java | 4 +-
.../config/lookup/BeanPropertyLookup.java | 17 +--
.../hashicorp/TestHashiCorpVaultConfiguration.java | 18 +--
...StandardHashiCorpVaultCommunicationService.java | 8 +-
.../src/main/asciidoc/administration-guide.adoc | 2 +-
nifi-docs/src/main/asciidoc/toolkit-guide.adoc | 46 ++++++-
.../NiFiPropertiesLoaderGroovyTest.groovy | 6 +-
.../resources/conf/bootstrap-hashicorp-vault.conf | 48 ++++++++
.../src/main/resources/conf/bootstrap.conf | 5 +
.../properties/NiFiRegistryPropertiesLoader.java | 5 +-
.../NiFiRegistryBootstrapUtilsGroovyTest.groovy | 4 +-
.../resources/conf/bootstrap-hashicorp-vault.conf | 48 ++++++++
.../src/main/resources/conf/bootstrap.conf | 7 +-
.../nifi/properties/ConfigEncryptionTool.groovy | 6 +-
.../encryptconfig/NiFiRegistryDecryptMode.groovy | 3 +-
.../toolkit/encryptconfig/NiFiRegistryMode.groovy | 26 +++-
30 files changed, 733 insertions(+), 131 deletions(-)
diff --git
a/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/AbstractBootstrapPropertiesLoader.java
b/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/AbstractBootstrapPropertiesLoader.java
index cbeea84..0bc378e 100644
---
a/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/AbstractBootstrapPropertiesLoader.java
+++
b/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/AbstractBootstrapPropertiesLoader.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.properties;
+import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -24,6 +25,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Objects;
import java.util.Properties;
/**
@@ -74,14 +76,27 @@ public abstract class AbstractBootstrapPropertiesLoader {
* @throws IOException If the file is not readable
*/
public BootstrapProperties loadBootstrapProperties(final String
bootstrapPath) throws IOException {
- final Properties properties = new Properties();
final Path bootstrapFilePath =
getBootstrapFile(bootstrapPath).toPath();
- try (final InputStream bootstrapInput =
Files.newInputStream(bootstrapFilePath)) {
+ return loadBootstrapProperties(bootstrapFilePath,
getApplicationPrefix());
+ }
+
+ /**
+ * Loads a properties file into a BootstrapProperties object.
+ * @param bootstrapPath The path to the properties file
+ * @param propertyPrefix The property prefix to enforce
+ * @return The BootstrapProperties
+ * @throws IOException If the properties file could not be read
+ */
+ public static BootstrapProperties loadBootstrapProperties(final Path
bootstrapPath, final String propertyPrefix) throws IOException {
+ Objects.requireNonNull(bootstrapPath, "Bootstrap path must be
provided");
+ Objects.requireNonNull(propertyPrefix, "Property prefix must be
provided");
+
+ final Properties properties = new Properties();
+ try (final InputStream bootstrapInput =
Files.newInputStream(bootstrapPath)) {
properties.load(bootstrapInput);
- return new BootstrapProperties(getApplicationPrefix(), properties,
bootstrapFilePath);
+ return new BootstrapProperties(propertyPrefix, properties,
bootstrapPath);
} catch (final IOException e) {
- logger.error("Cannot read from bootstrap.conf file at {}",
bootstrapFilePath);
- throw new IOException("Cannot read from bootstrap.conf", e);
+ throw new IOException("Cannot read from " + bootstrapPath, e);
}
}
@@ -97,7 +112,7 @@ public abstract class AbstractBootstrapPropertiesLoader {
public String extractKeyFromBootstrapFile(final String bootstrapPath)
throws IOException {
final BootstrapProperties bootstrapProperties =
loadBootstrapProperties(bootstrapPath);
- return bootstrapProperties.getBootstrapSensitiveKey().orElseGet(() -> {
+ return
bootstrapProperties.getProperty(BootstrapPropertyKey.SENSITIVE_KEY).orElseGet(()
-> {
logger.warn("No encryption key present in the bootstrap.conf file
at {}", bootstrapProperties.getConfigFilePath());
return "";
});
@@ -121,8 +136,7 @@ public abstract class AbstractBootstrapPropertiesLoader {
if (confDir.exists() && confDir.canRead()) {
expectedBootstrapFile = new File(confDir, BOOTSTRAP_CONF);
} else {
- logger.error("Cannot read from bootstrap.conf file at {} --
conf/ directory is missing or permissions are incorrect",
confDir.getAbsolutePath());
- throw new IOException("Cannot read from bootstrap.conf");
+ throw new IOException(String.format("Cannot read %s directory
for %s", confDir, bootstrapPath));
}
} else {
expectedBootstrapFile = new File(bootstrapPath);
@@ -131,8 +145,7 @@ public abstract class AbstractBootstrapPropertiesLoader {
if (expectedBootstrapFile.exists() && expectedBootstrapFile.canRead())
{
return expectedBootstrapFile;
} else {
- logger.error("Cannot read from bootstrap.conf file at {} -- file
is missing or permissions are incorrect",
expectedBootstrapFile.getAbsolutePath());
- throw new IOException("Cannot read from bootstrap.conf");
+ throw new IOException("Cannot read from " +
expectedBootstrapFile.getAbsolutePath());
}
}
diff --git
a/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/BootstrapProperties.java
b/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/BootstrapProperties.java
index 4713e27..bee82e1 100644
---
a/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/BootstrapProperties.java
+++
b/nifi-commons/nifi-property-utils/src/main/java/org/apache/nifi/properties/BootstrapProperties.java
@@ -17,17 +17,29 @@
package org.apache.nifi.properties;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Enumeration;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
+import java.util.Set;
/**
* Properties representing bootstrap.conf.
*/
public class BootstrapProperties extends StandardReadableProperties {
private static final String PROPERTY_KEY_FORMAT = "%s.%s";
- private static final String BOOTSTRAP_SENSITIVE_KEY =
"bootstrap.sensitive.key";
+
+ public enum BootstrapPropertyKey {
+ SENSITIVE_KEY("bootstrap.sensitive.key"),
+
HASHICORP_VAULT_SENSITIVE_PROPERTY_PROVIDER_CONF("bootstrap.protection.hashicorp.vault.conf");
+
+ private final String key;
+
+ BootstrapPropertyKey(final String key) {
+ this.key = key;
+ }
+ }
private final String propertyPrefix;
private final Path configFilePath;
@@ -44,6 +56,29 @@ public class BootstrapProperties extends
StandardReadableProperties {
}
/**
+ * Ensures that blank or empty properties are returned as null.
+ * @param key The property key
+ * @param defaultValue The default value to use if the value is null or
empty
+ * @return The property value (null if empty or blank)
+ */
+ @Override
+ public String getProperty(final String key, final String defaultValue) {
+ final String property = super.getProperty(key, defaultValue);
+ return isBlank(property) ? null : property;
+ }
+
+ /**
+ * Ensures that blank or empty properties are returned as null.
+ * @param key The property key
+ * @return The property value (null if empty or blank)
+ */
+ @Override
+ public String getProperty(final String key) {
+ final String property = super.getProperty(key);
+ return isBlank(property) ? null : property;
+ }
+
+ /**
* Returns the path to the bootstrap config file.
* @return The path to the file
*/
@@ -72,15 +107,45 @@ public class BootstrapProperties extends
StandardReadableProperties {
}
/**
- * Returns the bootstrap sensitive key.
- * @return The bootstrap sensitive key
+ * Returns the optional property value with the given BootstrapPropertyKey.
+ * @param key A BootstrapPropertyKey, representing properties in
bootstrap.conf
+ * @return The property value
*/
- public Optional<String> getBootstrapSensitiveKey() {
- return
Optional.ofNullable(getProperty(getPropertyKey(BOOTSTRAP_SENSITIVE_KEY)));
+ public Optional<String> getProperty(final BootstrapPropertyKey key) {
+ return Optional.ofNullable(getProperty(getPropertyKey(key.key)));
}
@Override
public String toString() {
return String.format("Bootstrap properties [%s] with prefix [%s]",
configFilePath, propertyPrefix);
}
+
+ /**
+ * An empty instance of BootstrapProperties.
+ */
+ public static final BootstrapProperties EMPTY = new
BootstrapProperties("", new Properties(), Paths.get("conf/bootstrap.conf")) {
+ @Override
+ public Set<String> getPropertyKeys() {
+ return null;
+ }
+
+ @Override
+ public String getProperty(String key) {
+ return null;
+ }
+
+ @Override
+ public String getProperty(String key, String defaultValue) {
+ return null;
+ }
+
+ @Override
+ public int size() {
+ return 0;
+ }
+ };
+
+ private static boolean isBlank(final String string) {
+ return (string == null) || string.isEmpty() || string.trim().isEmpty();
+ }
}
diff --git a/nifi-commons/nifi-sensitive-property-provider/pom.xml
b/nifi-commons/nifi-sensitive-property-provider/pom.xml
index bdfa9ba..59660e4 100644
--- a/nifi-commons/nifi-sensitive-property-provider/pom.xml
+++ b/nifi-commons/nifi-sensitive-property-provider/pom.xml
@@ -42,6 +42,17 @@
<artifactId>nifi-security-utils</artifactId>
<version>1.14.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-vault-utils</artifactId>
+ <version>1.14.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.10.0</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<!-- Required to run Groovy tests without any Java tests -->
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java
index 3999a3a..6d30375 100644
---
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AESSensitivePropertyProvider.java
@@ -81,7 +81,7 @@ public class AESSensitivePropertyProvider extends
AbstractSensitivePropertyProvi
}
@Override
- protected boolean isSupported(final BootstrapProperties
bootstrapProperties) {
+ public boolean isSupported() {
return true; // AES protection is always supported
}
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
new file mode 100644
index 0000000..3a06157
--- /dev/null
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractHashiCorpVaultSensitivePropertyProvider.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.properties;
+
+import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
+import org.apache.nifi.vault.hashicorp.HashiCorpVaultCommunicationService;
+import
org.apache.nifi.vault.hashicorp.StandardHashiCorpVaultCommunicationService;
+import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
+import
org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration.VaultConfigurationKey;
+import org.springframework.core.env.PropertySource;
+
+import java.io.IOException;
+import java.nio.file.Paths;
+
+public abstract class AbstractHashiCorpVaultSensitivePropertyProvider extends
AbstractSensitivePropertyProvider {
+ private static final String VAULT_PREFIX = "vault";
+
+ private final String path;
+ private final HashiCorpVaultCommunicationService vaultCommunicationService;
+ private final BootstrapProperties vaultBootstrapProperties;
+
+ AbstractHashiCorpVaultSensitivePropertyProvider(final BootstrapProperties
bootstrapProperties) {
+ super(bootstrapProperties);
+
+ final String vaultBootstrapConfFilename = bootstrapProperties
+
.getProperty(BootstrapPropertyKey.HASHICORP_VAULT_SENSITIVE_PROPERTY_PROVIDER_CONF).orElse(null);
+ vaultBootstrapProperties =
getVaultBootstrapProperties(vaultBootstrapConfFilename);
+ path = getSecretsEnginePath(vaultBootstrapProperties);
+ if (hasRequiredVaultProperties()) {
+ try {
+ vaultCommunicationService = new
StandardHashiCorpVaultCommunicationService(getVaultPropertySource(vaultBootstrapConfFilename));
+ } catch (IOException e) {
+ throw new SensitivePropertyProtectionException("Error
configuring HashiCorpVaultCommunicationService", e);
+ }
+ } else {
+ vaultCommunicationService = null;
+ }
+ }
+
+ /**
+ * Return the configured Secrets Engine path for this sensitive property
provider.
+ * @param vaultBootstrapProperties The Properties from the file located at
bootstrap.protection.hashicorp.vault.conf
+ * @return The Secrets Engine path
+ */
+ protected abstract String getSecretsEnginePath(final BootstrapProperties
vaultBootstrapProperties);
+
+ private static BootstrapProperties getVaultBootstrapProperties(final
String vaultBootstrapConfFilename) {
+ final BootstrapProperties vaultBootstrapProperties;
+ if (vaultBootstrapConfFilename != null) {
+ try {
+ vaultBootstrapProperties =
AbstractBootstrapPropertiesLoader.loadBootstrapProperties(
+ Paths.get(vaultBootstrapConfFilename), VAULT_PREFIX);
+ } catch (IOException e) {
+ throw new SensitivePropertyProtectionException("Could not load
" + vaultBootstrapConfFilename, e);
+ }
+ } else {
+ vaultBootstrapProperties = null;
+ }
+ return vaultBootstrapProperties;
+ }
+
+ private PropertySource<?> getVaultPropertySource(final String
vaultBootstrapConfFilename) throws IOException {
+ return
HashiCorpVaultConfiguration.createPropertiesFileSource(vaultBootstrapConfFilename);
+ }
+
+ /**
+ * Returns the Secrets Engine path.
+ * @return The Secrets Engine path
+ */
+ protected String getPath() {
+ return path;
+ }
+
+ protected HashiCorpVaultCommunicationService
getVaultCommunicationService() {
+ if (vaultCommunicationService == null) {
+ throw new SensitivePropertyProtectionException(getIdentifierKey()
+ " protection scheme is not fully configured in
hashicorp-vault-bootstrap.conf");
+ }
+ return vaultCommunicationService;
+ }
+
+ @Override
+ public boolean isSupported() {
+ return hasRequiredVaultProperties();
+ }
+
+ /**
+ * Returns the Vault-specific bootstrap properties (e.g.,
bootstrap-vault.properties)
+ * @return The Vault-specific bootstrap properties
+ */
+ protected BootstrapProperties getVaultBootstrapProperties() {
+ return vaultBootstrapProperties;
+ }
+
+ private boolean hasRequiredVaultProperties() {
+ return vaultBootstrapProperties != null
+ &&
(vaultBootstrapProperties.getProperty(VaultConfigurationKey.URI.getKey()) !=
null)
+ &&
hasRequiredSecretsEngineProperties(vaultBootstrapProperties);
+ }
+
+ /**
+ * Return true if the relevant Secrets Engine-specific properties are
configured.
+ * @param vaultBootstrapProperties The Vault-specific bootstrap properties
+ * @return true if the relevant Secrets Engine-specific properties are
configured
+ */
+ protected abstract boolean hasRequiredSecretsEngineProperties(final
BootstrapProperties vaultBootstrapProperties);
+
+ /**
+ * Returns the key used to identify the provider implementation in {@code
nifi.properties},
+ * in the format 'vault/{secretsEngine}/{secretsEnginePath}'.
+ *
+ * @return the key to persist in the sibling property
+ */
+ @Override
+ public String getIdentifierKey() {
+ return getProtectionScheme().getIdentifier(path);
+ }
+
+}
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractSensitivePropertyProvider.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractSensitivePropertyProvider.java
index b52fb73..e163747 100644
---
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractSensitivePropertyProvider.java
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/AbstractSensitivePropertyProvider.java
@@ -33,14 +33,6 @@ public abstract class AbstractSensitivePropertyProvider
implements SensitiveProp
*/
protected abstract PropertyProtectionScheme getProtectionScheme();
- /**
- * Return true if this SensitivePropertyProvider is supported, given the
provided
- * Bootstrap properties.
- * @param bootstrapProperties The Bootstrap properties
- * @return True if this SensitivePropertyProvider is supported
- */
- protected abstract boolean isSupported(BootstrapProperties
bootstrapProperties);
-
@Override
public String getName() {
return getProtectionScheme().getName();
@@ -55,9 +47,4 @@ public abstract class AbstractSensitivePropertyProvider
implements SensitiveProp
public String getIdentifierKey() {
return getProtectionScheme().getIdentifier();
}
-
- @Override
- public boolean isSupported() {
- return isSupported(bootstrapProperties);
- }
}
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
new file mode 100644
index 0000000..b670cb1
--- /dev/null
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/HashiCorpVaultTransitSensitivePropertyProvider.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.properties;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Uses the HashiCorp Vault Transit Secrets Engine to encrypt sensitive values
at rest.
+ */
+public class HashiCorpVaultTransitSensitivePropertyProvider extends
AbstractHashiCorpVaultSensitivePropertyProvider {
+ private static final Charset PROPERTY_CHARSET = StandardCharsets.UTF_8;
+ private static final String TRANSIT_PATH = "vault.transit.path";
+
+ HashiCorpVaultTransitSensitivePropertyProvider(final BootstrapProperties
bootstrapProperties) {
+ super(bootstrapProperties);
+ }
+
+ @Override
+ protected String getSecretsEnginePath(final BootstrapProperties
vaultBootstrapProperties) {
+ if (vaultBootstrapProperties == null) {
+ return null;
+ }
+ final String transitPath =
vaultBootstrapProperties.getProperty(TRANSIT_PATH);
+ // Validate transit path
+ try {
+
PropertyProtectionScheme.fromIdentifier(getProtectionScheme().getIdentifier(transitPath));
+ } catch (IllegalArgumentException e) {
+ throw new SensitivePropertyProtectionException(String.format("%s
[%s] contains unsupported characters", TRANSIT_PATH, transitPath), e);
+ }
+
+ return transitPath;
+ }
+
+ @Override
+ protected PropertyProtectionScheme getProtectionScheme() {
+ return PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT;
+ }
+
+ @Override
+ protected boolean hasRequiredSecretsEngineProperties(final
BootstrapProperties vaultBootstrapProperties) {
+ return getSecretsEnginePath(vaultBootstrapProperties) != null;
+ }
+
+ /**
+ * Returns the encrypted cipher text.
+ *
+ * @param unprotectedValue the sensitive value
+ * @return the value to persist in the {@code nifi.properties} file
+ * @throws SensitivePropertyProtectionException if there is an exception
encrypting the value
+ */
+ @Override
+ public String protect(final String unprotectedValue) throws
SensitivePropertyProtectionException {
+ if (StringUtils.isBlank(unprotectedValue)) {
+ throw new IllegalArgumentException("Cannot encrypt an empty
value");
+ }
+
+ return getVaultCommunicationService().encrypt(getPath(),
unprotectedValue.getBytes(PROPERTY_CHARSET));
+ }
+
+ /**
+ * Returns the decrypted plaintext.
+ *
+ * @param protectedValue the cipher text read from the {@code
nifi.properties} file
+ * @return the raw value to be used by the application
+ * @throws SensitivePropertyProtectionException if there is an error
decrypting the cipher text
+ */
+ @Override
+ public String unprotect(final String protectedValue) throws
SensitivePropertyProtectionException {
+ if (StringUtils.isBlank(protectedValue)) {
+ throw new IllegalArgumentException("Cannot decrypt an empty
value");
+ }
+
+ return new String(getVaultCommunicationService().decrypt(getPath(),
protectedValue), PROPERTY_CHARSET);
+ }
+}
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
index 4017406..8c321f2 100644
---
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/PropertyProtectionScheme.java
@@ -24,7 +24,8 @@ import java.util.Objects;
* SensitivePropertyProvider.
*/
public enum PropertyProtectionScheme {
- AES_GCM("aes/gcm/(128|192|256)", "aes/gcm/%s", "AES Sensitive Property
Provider", true);
+ AES_GCM("aes/gcm/(128|192|256)", "aes/gcm/%s", "AES Sensitive Property
Provider", true),
+ HASHICORP_VAULT_TRANSIT("hashicorp/vault/transit/[a-zA-Z0-9_-]+",
"hashicorp/vault/transit/%s", "HashiCorp Vault Transit Engine Sensitive
Property Provider", false);
PropertyProtectionScheme(final String identifierPattern, final String
identifierFormat, final String name, final boolean requiresSecretKey) {
this.identifierPattern = identifierPattern;
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
index ef59333..230bb22 100644
---
a/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
+++
b/nifi-commons/nifi-sensitive-property-provider/src/main/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactory.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.properties;
+import org.apache.nifi.properties.BootstrapProperties.BootstrapPropertyKey;
import org.apache.nifi.util.NiFiBootstrapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,7 +77,7 @@ public class StandardSensitivePropertyProviderFactory
implements SensitiveProper
}
private String getKeyHex() {
- return keyHex.orElseGet(() ->
getBootstrapProperties().getBootstrapSensitiveKey()
+ return keyHex.orElseGet(() ->
getBootstrapProperties().getProperty(BootstrapPropertyKey.SENSITIVE_KEY)
.orElseThrow(() -> new
SensitivePropertyProtectionException("Could not read root key from
bootstrap.conf")));
}
@@ -90,8 +91,8 @@ public class StandardSensitivePropertyProviderFactory
implements SensitiveProper
try {
return NiFiBootstrapUtils.loadBootstrapProperties();
} catch (final IOException e) {
- logger.error("Error extracting root key from bootstrap.conf
for login identity provider decryption", e);
- throw new SensitivePropertyProtectionException("Could not read
root key from bootstrap.conf");
+ logger.debug("Could not load bootstrap.conf from disk, so
using empty bootstrap.conf", e);
+ return BootstrapProperties.EMPTY;
}
});
}
@@ -104,7 +105,8 @@ public class StandardSensitivePropertyProviderFactory
implements SensitiveProper
switch (protectionScheme) {
case AES_GCM:
return providerMap.computeIfAbsent(protectionScheme, s -> new
AESSensitivePropertyProvider(keyHex));
- // Other providers may choose to pass getBootstrapProperties()
into the constructor
+ case HASHICORP_VAULT_TRANSIT:
+ return providerMap.computeIfAbsent(protectionScheme, s -> new
HashiCorpVaultTransitSensitivePropertyProvider(getBootstrapProperties()));
default:
throw new SensitivePropertyProtectionException("Unsupported
protection scheme " + protectionScheme);
}
diff --git
a/nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactoryTest.java
b/nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactoryTest.java
index f36f259..81995bf 100644
---
a/nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactoryTest.java
+++
b/nifi-commons/nifi-sensitive-property-provider/src/test/java/org/apache/nifi/properties/StandardSensitivePropertyProviderFactoryTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.nifi.properties;
+import org.apache.commons.io.FilenameUtils;
import org.apache.nifi.util.NiFiProperties;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.junit.AfterClass;
@@ -23,8 +24,10 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.internal.util.io.IOUtil;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.Security;
@@ -32,8 +35,10 @@ import java.util.Properties;
import java.util.function.Supplier;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
public class StandardSensitivePropertyProviderFactoryTest {
@@ -45,8 +50,9 @@ public class StandardSensitivePropertyProviderFactoryTest {
private static final String AD_HOC_KEY_HEX =
"123456789ABCDEFFEDCBA98765432101";
private static Path tempConfDir;
- private static Path mockBootstrapConf;
- private static Path mockNifiProperties;
+ private static Path bootstrapConf;
+ private static Path hashicorpVaultBootstrapConf;
+ private static Path nifiProperties;
private static NiFiProperties niFiProperties;
@@ -54,23 +60,28 @@ public class StandardSensitivePropertyProviderFactoryTest {
public static void initOnce() throws IOException {
Security.addProvider(new BouncyCastleProvider());
tempConfDir = Files.createTempDirectory("conf");
- mockBootstrapConf = Files.createTempFile("bootstrap",
".conf").toAbsolutePath();
+ bootstrapConf = Files.createTempFile("bootstrap",
".conf").toAbsolutePath();
+ hashicorpVaultBootstrapConf =
Files.createTempFile("bootstrap-hashicorp-vault", ".conf").toAbsolutePath();
- mockNifiProperties = Files.createTempFile("nifi",
".properties").toAbsolutePath();
+ nifiProperties = Files.createTempFile("nifi",
".properties").toAbsolutePath();
- mockBootstrapConf = Files.move(mockBootstrapConf,
tempConfDir.resolve("bootstrap.conf"));
- mockNifiProperties = Files.move(mockNifiProperties,
tempConfDir.resolve("nifi.properties"));
+ bootstrapConf = Files.move(bootstrapConf,
tempConfDir.resolve("bootstrap.conf"));
+ nifiProperties = Files.move(nifiProperties,
tempConfDir.resolve("nifi.properties"));
- IOUtil.writeText("nifi.bootstrap.sensitive.key=" + BOOTSTRAP_KEY_HEX,
mockBootstrapConf.toFile());
- System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH,
mockNifiProperties.toString());
+ final String bootstrapConfText = String.format("%s=%s\n%s=%s",
+ "nifi.bootstrap.sensitive.key", BOOTSTRAP_KEY_HEX,
+ "nifi.bootstrap.protection.hashicorp.vault.conf",
FilenameUtils.separatorsToUnix(hashicorpVaultBootstrapConf.toString()));
+ IOUtil.writeText(bootstrapConfText, bootstrapConf.toFile());
+ System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH,
FilenameUtils.separatorsToUnix(nifiProperties.toString()));
niFiProperties = new NiFiProperties();
}
@AfterClass
public static void tearDownOnce() throws IOException {
- Files.deleteIfExists(mockBootstrapConf);
- Files.deleteIfExists(mockNifiProperties);
+ Files.deleteIfExists(bootstrapConf);
+ Files.deleteIfExists(hashicorpVaultBootstrapConf);
+ Files.deleteIfExists(nifiProperties);
Files.deleteIfExists(tempConfDir);
System.clearProperty(NiFiProperties.PROPERTIES_FILE_PATH);
}
@@ -99,12 +110,62 @@ public class StandardSensitivePropertyProviderFactoryTest {
private Supplier<BootstrapProperties> mockBootstrapProperties() throws
IOException {
final Properties bootstrapProperties = new Properties();
- try (final InputStream inputStream =
Files.newInputStream(mockBootstrapConf)) {
+ try (final InputStream inputStream =
Files.newInputStream(bootstrapConf)) {
bootstrapProperties.load(inputStream);
- return () -> new BootstrapProperties("nifi", bootstrapProperties,
mockBootstrapConf);
+ return () -> new BootstrapProperties("nifi", bootstrapProperties,
bootstrapConf);
}
}
+ private void configureHashicorpVault(final Properties properties) throws
IOException {
+ try (OutputStream out = new
FileOutputStream(hashicorpVaultBootstrapConf.toFile())) {
+ properties.store(out, "HashiCorpVault test");
+ }
+ }
+
+ @Test
+ public void testHashicorpVaultTransit() throws IOException {
+ configureDefaultFactory();
+ final Properties properties = new Properties();
+ properties.put("vault.transit.path", "nifi-transit");
+ configureHashicorpVault(properties);
+
+ final SensitivePropertyProvider spp =
factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
+ }
+
+ @Test
+ public void testHashicorpVaultTransit_isSupported() throws IOException {
+ configureDefaultFactory();
+ final Properties properties = new Properties();
+ properties.put("vault.transit.path", "nifi-transit");
+ properties.put("vault.uri", "http://localhost:8200";);
+ properties.put("vault.token", "test-token");
+ configureHashicorpVault(properties);
+
+ SensitivePropertyProvider spp =
factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
+ assertTrue(spp.isSupported());
+
+ properties.remove("vault.uri");
+ configureHashicorpVault(properties);
+ configureDefaultFactory();
+ spp =
factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
+ assertFalse(spp.isSupported());
+
+ properties.put("vault.uri", "http://localhost:8200";);
+ properties.remove("vault.transit.path");
+ spp =
factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT);
+ assertFalse(spp.isSupported());
+ }
+
+ @Test
+ public void testHashicorpVaultTransit_invalidCharacters() throws
IOException {
+ configureDefaultFactory();
+ final Properties properties = new Properties();
+ properties.put("vault.transit.path", "invalid/characters");
+ configureHashicorpVault(properties);
+
+ assertThrows(SensitivePropertyProtectionException.class, () ->
factory.getProvider(PropertyProtectionScheme.HASHICORP_VAULT_TRANSIT));
+ }
+
@Test
public void testAES_GCM() throws IOException {
configureDefaultFactory();
diff --git
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
index 8f92fb3..61f0176 100644
---
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
+++
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/StandardHashiCorpVaultCommunicationService.java
@@ -16,27 +16,21 @@
*/
package org.apache.nifi.vault.hashicorp;
-import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperties;
+import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultPropertySource;
+import org.springframework.core.env.PropertySource;
import org.springframework.vault.authentication.SimpleSessionManager;
import org.springframework.vault.client.ClientHttpRequestFactoryFactory;
import org.springframework.vault.core.VaultTemplate;
import org.springframework.vault.core.VaultTransitOperations;
import org.springframework.vault.support.Ciphertext;
-import org.springframework.vault.support.ClientOptions;
import org.springframework.vault.support.Plaintext;
-import org.springframework.vault.support.SslConfiguration;
-
-import java.time.Duration;
-import java.util.Optional;
-import java.util.concurrent.TimeUnit;
/**
* Implements the VaultCommunicationService using Spring Vault
*/
public class StandardHashiCorpVaultCommunicationService implements
HashiCorpVaultCommunicationService {
- private static final String HTTPS = "https";
private final HashiCorpVaultConfiguration vaultConfiguration;
private final VaultTemplate vaultTemplate;
@@ -44,42 +38,26 @@ public class StandardHashiCorpVaultCommunicationService
implements HashiCorpVaul
/**
* Creates a VaultCommunicationService that uses Spring Vault.
- * @param vaultProperties Properties to configure the service
+ * @param propertySources Property sources to configure the service
* @throws HashiCorpVaultConfigurationException If the configuration was
invalid
*/
- public StandardHashiCorpVaultCommunicationService(final
HashiCorpVaultProperties vaultProperties) throws
HashiCorpVaultConfigurationException {
- this.vaultConfiguration = new
HashiCorpVaultConfiguration(vaultProperties);
-
- final SslConfiguration sslConfiguration =
vaultProperties.getUri().contains(HTTPS)
- ? vaultConfiguration.sslConfiguration() :
SslConfiguration.unconfigured();
-
- final ClientOptions clientOptions = getClientOptions(vaultProperties);
+ public StandardHashiCorpVaultCommunicationService(final
PropertySource<?>... propertySources) throws
HashiCorpVaultConfigurationException {
+ vaultConfiguration = new HashiCorpVaultConfiguration(propertySources);
vaultTemplate = new VaultTemplate(vaultConfiguration.vaultEndpoint(),
- ClientHttpRequestFactoryFactory.create(clientOptions,
sslConfiguration),
+
ClientHttpRequestFactoryFactory.create(vaultConfiguration.clientOptions(),
vaultConfiguration.sslConfiguration()),
new
SimpleSessionManager(vaultConfiguration.clientAuthentication()));
transitOperations = vaultTemplate.opsForTransit();
}
- private static ClientOptions getClientOptions(HashiCorpVaultProperties
vaultProperties) {
- final ClientOptions clientOptions = new ClientOptions();
- Duration readTimeoutDuration = clientOptions.getReadTimeout();
- Duration connectionTimeoutDuration =
clientOptions.getConnectionTimeout();
- final Optional<String> configuredReadTimeout =
vaultProperties.getReadTimeout();
- if (configuredReadTimeout.isPresent()) {
- readTimeoutDuration = getDuration(configuredReadTimeout.get());
- }
- final Optional<String> configuredConnectionTimeout =
vaultProperties.getConnectionTimeout();
- if (configuredConnectionTimeout.isPresent()) {
- connectionTimeoutDuration =
getDuration(configuredConnectionTimeout.get());
- }
- return new ClientOptions(connectionTimeoutDuration,
readTimeoutDuration);
- }
-
- private static Duration getDuration(String formattedDuration) {
- final double duration =
FormatUtils.getPreciseTimeDuration(formattedDuration, TimeUnit.MILLISECONDS);
- return Duration.ofMillis(Double.valueOf(duration).longValue());
+ /**
+ * Creates a VaultCommunicationService that uses Spring Vault.
+ * @param vaultProperties Properties to configure the service
+ * @throws HashiCorpVaultConfigurationException If the configuration was
invalid
+ */
+ public StandardHashiCorpVaultCommunicationService(final
HashiCorpVaultProperties vaultProperties) throws
HashiCorpVaultConfigurationException {
+ this(new HashiCorpVaultPropertySource(vaultProperties));
}
@Override
diff --git
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultConfiguration.java
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultConfiguration.java
index 44ad4e6..34ce6c6 100644
---
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultConfiguration.java
+++
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultConfiguration.java
@@ -16,31 +16,124 @@
*/
package org.apache.nifi.vault.hashicorp.config;
+import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.vault.hashicorp.HashiCorpVaultConfigurationException;
import org.springframework.core.env.ConfigurableEnvironment;
+import org.springframework.core.env.PropertySource;
import org.springframework.core.env.StandardEnvironment;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.support.ResourcePropertySource;
+import org.springframework.vault.client.RestTemplateFactory;
import org.springframework.vault.config.EnvironmentVaultConfiguration;
+import org.springframework.vault.support.ClientOptions;
+import org.springframework.vault.support.SslConfiguration;
import java.io.IOException;
import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
/**
* A Vault configuration that uses the NiFiVaultEnvironment.
*/
public class HashiCorpVaultConfiguration extends EnvironmentVaultConfiguration
{
+ public enum VaultConfigurationKey {
+ AUTHENTICATION_PROPERTIES_FILE("vault.authentication.properties.file"),
+ READ_TIMEOUT("vault.read.timeout"),
+ CONNECTION_TIMEOUT("vault.connection.timeout"),
+ URI("vault.uri");
- public HashiCorpVaultConfiguration(final HashiCorpVaultProperties
vaultProperties) throws HashiCorpVaultConfigurationException {
+ private final String key;
+
+ VaultConfigurationKey(final String key) {
+ this.key = key;
+ }
+
+ /**
+ * Returns the property key.
+ * @return The property key
+ */
+ public String getKey() {
+ return key;
+ }
+ }
+
+ private static final String HTTPS = "https";
+
+ private final SslConfiguration sslConfiguration;
+ private final ClientOptions clientOptions;
+
+ /**
+ * Creates a HashiCorpVaultConfiguration from property sources
+ * @param propertySources A series of Spring PropertySource objects
+ * @throws HashiCorpVaultConfigurationException If the authentication
properties file could not be read
+ */
+ public HashiCorpVaultConfiguration(final PropertySource<?>...
propertySources) {
final ConfigurableEnvironment env = new StandardEnvironment();
+ for(final PropertySource<?> propertySource : propertySources) {
+ env.getPropertySources().addFirst(propertySource);
+ }
- try {
- env.getPropertySources().addFirst(new ResourcePropertySource(new
FileSystemResource(Paths.get(vaultProperties.getAuthPropertiesFilename()))));
- } catch (IOException e) {
- throw new HashiCorpVaultConfigurationException("Could not load
auth properties", e);
+ if
(env.containsProperty(VaultConfigurationKey.AUTHENTICATION_PROPERTIES_FILE.key))
{
+ final String authPropertiesFilename =
env.getProperty(VaultConfigurationKey.AUTHENTICATION_PROPERTIES_FILE.key);
+ try {
+ final PropertySource<?> authPropertiesSource =
createPropertiesFileSource(authPropertiesFilename);
+ env.getPropertySources().addFirst(authPropertiesSource);
+ } catch (IOException e) {
+ throw new HashiCorpVaultConfigurationException("Could not load
HashiCorp Vault authentication properties " + authPropertiesFilename, e);
+ }
}
- env.getPropertySources().addFirst(new
HashiCorpVaultPropertySource(vaultProperties));
this.setApplicationContext(new HashiCorpVaultApplicationContext(env));
+
+ sslConfiguration =
env.getProperty(VaultConfigurationKey.URI.key).contains(HTTPS)
+ ? super.sslConfiguration() : SslConfiguration.unconfigured();
+
+ clientOptions = getClientOptions();
+ }
+
+ /**
+ * A convenience method to create a PropertySource from a file on disk.
+ * @param filename The properties filename.
+ * @return A PropertySource containing the properties in the given file
+ * @throws IOException If the file could not be read
+ */
+ public static PropertySource<?> createPropertiesFileSource(final String
filename) throws IOException {
+ return new ResourcePropertySource(new
FileSystemResource(Paths.get(filename)));
+ }
+
+ @Override
+ public ClientOptions clientOptions() {
+ return clientOptions;
+ }
+
+ @Override
+ protected RestTemplateFactory getRestTemplateFactory() {
+ return this.restTemplateFactory(clientHttpRequestFactoryWrapper());
+ }
+
+ @Override
+ public SslConfiguration sslConfiguration() {
+ return sslConfiguration;
+ }
+
+ private ClientOptions getClientOptions() {
+ final ClientOptions clientOptions = new ClientOptions();
+ Duration readTimeoutDuration = clientOptions.getReadTimeout();
+ Duration connectionTimeoutDuration =
clientOptions.getConnectionTimeout();
+ final String configuredReadTimeout =
getEnvironment().getProperty(VaultConfigurationKey.READ_TIMEOUT.key);
+ if (configuredReadTimeout != null) {
+ readTimeoutDuration = getDuration(configuredReadTimeout);
+ }
+ final String configuredConnectionTimeout =
getEnvironment().getProperty(VaultConfigurationKey.CONNECTION_TIMEOUT.key);
+ if (configuredConnectionTimeout != null) {
+ connectionTimeoutDuration =
getDuration(configuredConnectionTimeout);
+ }
+ return new ClientOptions(connectionTimeoutDuration,
readTimeoutDuration);
+ }
+
+ private static Duration getDuration(String formattedDuration) {
+ final double duration =
FormatUtils.getPreciseTimeDuration(formattedDuration, TimeUnit.MILLISECONDS);
+ return Duration.ofMillis(Double.valueOf(duration).longValue());
}
}
diff --git
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperties.java
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperties.java
index 867ca0c..3a84bc9 100644
---
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperties.java
+++
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperties.java
@@ -79,6 +79,7 @@ public class HashiCorpVaultProperties {
return ssl;
}
+ @HashiCorpVaultProperty(key = "authentication.properties.file")
public String getAuthPropertiesFilename() {
return authPropertiesFilename;
}
diff --git
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperty.java
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperty.java
index 81bd87e..dcee774 100644
---
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperty.java
+++
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultProperty.java
@@ -28,4 +28,5 @@ import java.lang.annotation.Target;
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface HashiCorpVaultProperty {
+ String key() default "";
}
diff --git
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultPropertySource.java
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultPropertySource.java
index 446efc1..8e64d08 100644
---
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultPropertySource.java
+++
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/HashiCorpVaultPropertySource.java
@@ -30,10 +30,10 @@ public class HashiCorpVaultPropertySource extends
PropertySource<HashiCorpVaultP
private PropertyLookup propertyLookup;
- public HashiCorpVaultPropertySource(HashiCorpVaultProperties source) {
+ public HashiCorpVaultPropertySource(final HashiCorpVaultProperties source)
{
super(HashiCorpVaultPropertySource.class.getName(), source);
- propertyLookup = new BeanPropertyLookup(PREFIX,
HashiCorpVaultProperties.class, HashiCorpVaultProperty.class);
+ propertyLookup = new BeanPropertyLookup(PREFIX,
HashiCorpVaultProperties.class);
}
@Override
diff --git
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/lookup/BeanPropertyLookup.java
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/lookup/BeanPropertyLookup.java
index 2ad1ac1..da873af 100644
---
a/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/lookup/BeanPropertyLookup.java
+++
b/nifi-commons/nifi-vault-utils/src/main/java/org/apache/nifi/vault/hashicorp/config/lookup/BeanPropertyLookup.java
@@ -17,10 +17,10 @@
package org.apache.nifi.vault.hashicorp.config.lookup;
import org.apache.nifi.vault.hashicorp.HashiCorpVaultConfigurationException;
+import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperty;
import org.springframework.beans.BeanUtils;
import java.beans.PropertyDescriptor;
-import java.lang.annotation.Annotation;
import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.Map;
@@ -34,24 +34,25 @@ public class BeanPropertyLookup extends PropertyLookup {
private final Map<String, PropertyLookup> propertyLookupMap;
- public BeanPropertyLookup(final String prefix, final Class<?> beanClass,
final Class<? extends Annotation> propertyFilter) {
- this(prefix, beanClass, propertyFilter, null);
+ public BeanPropertyLookup(final String prefix, final Class<?> beanClass) {
+ this(prefix, beanClass, null);
}
- private BeanPropertyLookup(final String prefix, final Class<?> beanClass,
final Class<? extends Annotation> propertyFilter,
- final PropertyDescriptor propertyDescriptor) {
+ private BeanPropertyLookup(final String prefix, final Class<?> beanClass,
final PropertyDescriptor propertyDescriptor) {
super(propertyDescriptor);
propertyLookupMap =
Arrays.stream(BeanUtils.getPropertyDescriptors(beanClass))
- .filter(pd -> pd.getReadMethod().getAnnotation(propertyFilter)
!= null)
+ .filter(pd ->
pd.getReadMethod().getAnnotation(HashiCorpVaultProperty.class) != null)
.collect(Collectors.toMap(
pd -> getPropertyKey(prefix, pd),
pd ->
pd.getReadMethod().getReturnType().equals(String.class) ? new
ValuePropertyLookup(pd)
- : new
BeanPropertyLookup(getPropertyKey(prefix, pd),
pd.getReadMethod().getReturnType(), propertyFilter, pd)
+ : new
BeanPropertyLookup(getPropertyKey(prefix, pd),
pd.getReadMethod().getReturnType(), pd)
));
}
private static String getPropertyKey(final String prefix, final
PropertyDescriptor propertyDescriptor) {
- return prefix == null ? propertyDescriptor.getDisplayName() :
String.join(SEPARATOR, prefix, propertyDescriptor.getDisplayName());
+ final HashiCorpVaultProperty propertyAnnotation =
propertyDescriptor.getReadMethod().getAnnotation(HashiCorpVaultProperty.class);
+ final String unqualifiedPropertyKey =
!propertyAnnotation.key().isEmpty() ? propertyAnnotation.key() :
propertyDescriptor.getDisplayName();
+ return prefix == null ? unqualifiedPropertyKey: String.join(SEPARATOR,
prefix, unqualifiedPropertyKey);
}
@Override
diff --git
a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultConfiguration.java
b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultConfiguration.java
index 38cb2d9..e1ec59c 100644
---
a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultConfiguration.java
+++
b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultConfiguration.java
@@ -18,6 +18,7 @@ package org.apache.nifi.vault.hashicorp;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultConfiguration;
import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultProperties;
+import org.apache.nifi.vault.hashicorp.config.HashiCorpVaultPropertySource;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
@@ -113,13 +114,13 @@ public class TestHashiCorpVaultConfiguration {
}
}
- public void runTest() {
- config = new HashiCorpVaultConfiguration(propertiesBuilder.build());
+ public void runTest(final String expectedScheme) {
+ config = new HashiCorpVaultConfiguration(new
HashiCorpVaultPropertySource(propertiesBuilder.build()));
VaultEndpoint endpoint = config.vaultEndpoint();
Assert.assertEquals("localhost", endpoint.getHost());
Assert.assertEquals(8200, endpoint.getPort());
- Assert.assertEquals("http", endpoint.getScheme());
+ Assert.assertEquals(expectedScheme, endpoint.getScheme());
ClientAuthentication clientAuthentication =
config.clientAuthentication();
Assert.assertNotNull(clientAuthentication);
@@ -127,7 +128,7 @@ public class TestHashiCorpVaultConfiguration {
@Test
public void testBasicProperties() {
- this.runTest();
+ this.runTest("http");
}
@Test
@@ -140,8 +141,9 @@ public class TestHashiCorpVaultConfiguration {
propertiesBuilder.setTrustStoreType(TRUSTSTORE_TYPE_VALUE);
propertiesBuilder.setEnabledTlsProtocols(TLS_V_1_3_VALUE);
propertiesBuilder.setEnabledTlsCipherSuites(TEST_CIPHER_SUITE_VALUE);
+ propertiesBuilder.setUri(URI_VALUE.replace("http", "https"));
- this.runTest();
+ this.runTest("https");
SslConfiguration sslConfiguration = config.sslConfiguration();
Assert.assertEquals(keystoreFile.toFile().getAbsolutePath(),
sslConfiguration.getKeyStoreConfiguration().getResource().getFile().getAbsolutePath());
@@ -157,7 +159,7 @@ public class TestHashiCorpVaultConfiguration {
@Test
public void testInvalidTLS() {
propertiesBuilder.setUri(URI_VALUE.replace("http", "https"));
- Assert.assertThrows(NullPointerException.class, () -> this.runTest());
+ Assert.assertThrows(NullPointerException.class, () ->
this.runTest("https"));
}
@Test
@@ -169,7 +171,7 @@ public class TestHashiCorpVaultConfiguration {
authProperties = writeVaultAuthProperties(props);
propertiesBuilder.setAuthPropertiesFilename(authProperties.getAbsolutePath());
- Assert.assertThrows(IllegalArgumentException.class, () ->
this.runTest());
+ Assert.assertThrows(IllegalArgumentException.class, () ->
this.runTest("http"));
} finally {
if (authProperties != null) {
Files.deleteIfExists(authProperties.toPath());
@@ -185,7 +187,7 @@ public class TestHashiCorpVaultConfiguration {
authProperties = writeVaultAuthProperties(props);
propertiesBuilder.setAuthPropertiesFilename(authProperties.getAbsolutePath());
- Assert.assertThrows(IllegalArgumentException.class, () ->
this.runTest());
+ Assert.assertThrows(IllegalArgumentException.class, () ->
this.runTest("http"));
} finally {
if (authProperties != null) {
Files.deleteIfExists(authProperties.toPath());
diff --git
a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestStandardHashiCorpVaultCommunicationService.java
b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestStandardHashiCorpVaultCommunicationService.java
index f9b102a..6527ca2 100644
---
a/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestStandardHashiCorpVaultCommunicationService.java
+++
b/nifi-commons/nifi-vault-utils/src/test/java/org/apache/nifi/vault/hashicorp/TestStandardHashiCorpVaultCommunicationService.java
@@ -70,9 +70,8 @@ public class TestStandardHashiCorpVaultCommunicationService {
// Once to check if the URI is https, and once by VaultTemplate
Mockito.verify(properties, Mockito.times(2)).getUri();
- // Once each to check if they are configured
- Mockito.verify(properties, Mockito.times(1)).getConnectionTimeout();
- Mockito.verify(properties, Mockito.times(1)).getReadTimeout();
+ // Once to check if the property is set, and once to retrieve the value
+ Mockito.verify(properties,
Mockito.times(2)).getAuthPropertiesFilename();
// These should not be called because TLS is not configured
this.ensureTlsPropertiesAccessed(0);
@@ -94,9 +93,6 @@ public class TestStandardHashiCorpVaultCommunicationService {
Mockito.when(properties.getConnectionTimeout()).thenReturn(Optional.of("20
secs"));
Mockito.when(properties.getReadTimeout()).thenReturn(Optional.of("40
secs"));
this.configureService();
-
- Mockito.verify(properties, Mockito.times(1)).getConnectionTimeout();
- Mockito.verify(properties, Mockito.times(1)).getReadTimeout();
}
@Test
diff --git a/nifi-docs/src/main/asciidoc/administration-guide.adoc
b/nifi-docs/src/main/asciidoc/administration-guide.adoc
index 2e9e423..98146ab 100644
--- a/nifi-docs/src/main/asciidoc/administration-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/administration-guide.adoc
@@ -1759,7 +1759,7 @@ All options require a password
(`nifi.sensitive.props.key` value) of *at least 1
[[encrypt-config_tool]]
== Encrypted Passwords in Configuration Files
-In order to facilitate the secure setup of NiFi, you can use the
`encrypt-config` command line utility to encrypt raw configuration values that
NiFi decrypts in memory on startup. This extensible protection scheme
transparently allows NiFi to use raw values in operation, while protecting them
at rest. In the future, hardware security modules (HSM) and external secure
storage mechanisms will be integrated, but for now, an AES encryption provider
is the default implementation.
+In order to facilitate the secure setup of NiFi, you can use the
`encrypt-config` command line utility to encrypt raw configuration values that
NiFi decrypts in memory on startup. This extensible protection scheme
transparently allows NiFi to use raw values in operation, while protecting them
at rest. In addition to the default AES encryption provider, a HashiCorp Vault
encryption provider can be configured in the
`bootstrap-hashicorp-vault.properties` file.
This is a change in behavior; prior to 1.0, all configuration values were
stored in plaintext on the file system. POSIX file permissions were recommended
to limit unauthorized access to these files.
diff --git a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
index b59d45c..bdfc282 100644
--- a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
+++ b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc
@@ -434,13 +434,15 @@ The following are available options when targeting NiFi:
* `-u`,`--outputAuthorizers <file>` The destination
_authorizers.xml_ file containing protected config values (will not modify
input _authorizers.xml_)
* `-f`,`--flowXml <file>` The _flow.xml.gz_ file
currently protected with old password (will be overwritten unless `-g` is
specified)
* `-g`,`--outputFlowXml <file>` The destination _flow.xml.gz_
file containing protected config values (will not modify input _flow.xml.gz_)
- * `-b`,`--bootstrapConf <file>` The _bootstrap.conf_ file to
persist root key
+ * `-b`,`--bootstrapConf <file>` The bootstrap.conf file to
persist root key and to optionally provide any configuration for the protection
scheme.
+ * `-S`,`--protectionScheme <protectionScheme>` Selects the protection scheme
for encrypted properties. Valid values are: [AES_GCM, HASHICORP_VAULT_TRANSIT]
(default is AES_GCM)
* `-k`,`--key <keyhex>` The raw hexadecimal key to
use to encrypt the sensitive properties
* `-e`,`--oldKey <keyhex>` The old raw hexadecimal key
to use during key migration
+ * `-H`,`--oldProtectionScheme <protectionScheme>` The old protection scheme
to use during encryption migration (see --protectionScheme for possible
values). Default is AES_GCM
* `-p`,`--password <password>` The password from which to
derive the key to use to encrypt the sensitive properties
* `-w`,`--oldPassword <password>` The old password from which to
derive the key during migration
* `-r`,`--useRawKey` If provided, the secure
console will prompt for the raw key value in hexadecimal form
- * `-m`,`--migrate` If provided, the
_nifi.properties_ and/or _login-identity-providers.xml_ sensitive properties
will be re-encrypted with a new key
+ * `-m`,`--migrate` If provided, the
_nifi.properties_ and/or _login-identity-providers.xml_ sensitive properties
will be re-encrypted with the new scheme
* `-x`,`--encryptFlowXmlOnly` If provided, the properties
in _flow.xml.gz_ will be re-encrypted with a new key but the _nifi.properties_
and/or _login-identity-providers.xml_ files will not be modified
* `-s`,`--propsKey <password|keyhex>` The password or key to use to
encrypt the sensitive processor properties in _flow.xml.gz_
* `-A`,`--newFlowAlgorithm <algorithm>` The algorithm to use to
encrypt the sensitive processor properties in _flow.xml.gz_
@@ -454,9 +456,11 @@ The following are available options when targeting NiFi
Registry using the `--ni
* `-v`,`--verbose` Sets verbose mode (default
false)
* `-p`,`--password <password>` Protect the files using a
password-derived key. If an argument is not provided to this flag, interactive
mode will be triggered to prompt the user to enter the password.
* `-k`,`--key <keyhex>` Protect the files using a raw
hexadecimal key. If an argument is not provided to this flag, interactive mode
will be triggered to prompt the user to enter the key.
+ * `-S`,`--protectionScheme <protectionScheme>` Selects the protection scheme
for encrypted properties. Valid values are: [AES_GCM, HASHICORP_VAULT_TRANSIT]
(default is AES_GCM)
* `--oldPassword <password>` If the input files are
already protected using a password-derived key, this specifies the old password
so that the files can be unprotected before re-protecting.
* `--oldKey <keyhex>` If the input files are
already protected using a key, this specifies the raw hexadecimal key so that
the files can be unprotected before re-protecting.
- * `-b`,`--bootstrapConf <file>` The _bootstrap.conf_ file
containing no root key or an existing root key. If a new password or key is
specified (using `-p` or `-k`) and no output _bootstrap.conf_ file is
specified, then this file will be overwritten to persist the new root key.
+ * `-H`,`--oldProtectionScheme <protectionScheme>`The old protection scheme to
use during encryption migration (see --protectionScheme for possible values).
Default is AES_GCM.
+ * `-b`,`--bootstrapConf <file>` The _bootstrap.conf_ file
containing no root key or an existing root key, and any other protection scheme
configuration properties. If a new password or key is specified (using -p or
-k) and no output _bootstrap.conf_ file is specified, then this file will be
overwritten to persist the new master key.
* `-B`,`--outputBootstrapConf <file>` The destination
_bootstrap.conf_ file to persist root key. If specified, the input
_bootstrap.conf_ will not be modified.
* `-r`,`--nifiRegistryProperties <file>` The
_nifi-registry.properties_ file containing unprotected config values,
overwritten if no output file specified.
* `-R`,`--outputNifiRegistryProperties <file>` The destination
_nifi-registry.properties_ file containing protected config values.
@@ -466,6 +470,40 @@ The following are available options when targeting NiFi
Registry using the `--ni
* `-I`,`--outputIdentityProvidersXml <file>` The destination
_identity-providers.xml_ file containing protected config values.
* `--decrypt` Can be used with `-r` to
decrypt a previously encrypted NiFi Registry Properties file. Decrypted content
is printed to STDOUT.
+=== Protection Schemes
+The protection scheme can be selected during encryption using the
`--protectionScheme` flag. During migration, the former protection scheme is
specified using the `--oldProtectionScheme` flag. This distinction allows a
set of protected configuration files to be migrated not only to a new key, but
to a completely different protection scheme.
+
+==== AES_GCM
+The default protection scheme, `AES-G/CM` simply encrypts sensitive properties
and marks their protection as either `aes/gcm/256` or `aes/gcm/256` as
appropriate. This protection is all done within NiFi itself.
+
+==== HASHICORP_VAULT_TRANSIT
+This protection scheme uses HashiCorp Vault's Transit Secrets Engine
(https://www.vaultproject.io/docs/secrets/transit) to outsource encryption to a
configured Vault server. All HashiCorp Vault configuration is stored in the
`bootstrap-hashicorp-vault.conf` file, as referenced in the `bootstrap.conf` of
a NiFi or NiFi Registry instance. Therefore, when using the
HASHICORP_VAULT_TRANSIT protection scheme, the
`nifi(.registry)?.bootstrap.protection.hashicorp.vault.conf` property in the `b
[...]
+
+===== Required properties
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`vault.uri`|The HashiCorp Vault URI (e.g., `https://vault-server:8200`). If
not set, this provider will be disabled.|_none_
+|`vault.authentication.properties.file`|Filename of a properties file
containing Vault authentication properties. See the `Authentication-specific
property keys` section of
https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration
for all authentication property keys. If not set, all Spring Vault
authentication properties must be configured directly in
bootstrap-hashicorp-vault.conf.|_none_
+|`vault.transit.path`|The HashiCorp Vault `path` specifying the Transit
Secrets Engine (e.g., `nifi-transit`). Valid characters include alphanumeric,
dash, and underscore. If not set, this provider will be disabled.|_none_
+|===
+
+===== Optional properties
+[options="header,footer"]
+|===
+|Property Name|Description|Default
+|`vault.connection.timeout`|The connection timeout of the Vault client|`5 secs`
+|`vault.read.timeout`|The read timeout of the Vault client|`15 secs`
+|`vault.ssl.enabledCipherSuites`|A comma-separated list of the enabled TLS
cipher suites|_none_
+|`vault.ssl.enabledProtocols`|A comma-separated list of the enabled TLS
protocols|_none_
+|`vault.ssl.key-store`|Path to a keystore. Required if the Vault server is
TLS-enabled|_none_
+|`vault.ssl.key-store-type`|Keystore type (JKS, BCFKS or PKCS12). Required if
the Vault server is TLS-enabled|_none_
+|`vault.ssl.key-store-password`|Keystore password. Required if the Vault
server is TLS-enabled|_none_
+|`vault.ssl.trust-store`|Path to a truststore. Required if the Vault server
is TLS-enabled|_none_
+|`vault.ssl.trust-store-type`|Truststore type (JKS, BCFKS or PKCS12).
Required if the Vault server is TLS-enabled|_none_
+|`vault.ssl.trust-store-password`|Truststore password. Required if the Vault
server is TLS-enabled|_none_
+|===
+
=== Examples
==== NiFi
@@ -658,6 +696,8 @@ for each phase (old vs. new), and any combination is
sufficient:
* old password -> new key
* old password -> new password
+In order to change the protection scheme (e.g., migrating from AES encryption
to Vault encryption), specify the `--protectionScheme`
+and `--oldProtectionScheme` in the migration command.
== File Manager
The File Manager utility (invoked as `./bin/file-manager.sh` or
`bin\file-manager.bat`) allows system administrators to take a backup of an
existing NiFi installation, install a new version of NiFi in a designated
location (while migrating any previous configuration settings) or restore an
installation from a previous backup. File Manager supports NiFi version 1.0.0
and higher.
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
index b73a970..4809923 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-properties-loader/src/test/groovy/org/apache/nifi/properties/NiFiPropertiesLoaderGroovyTest.groovy
@@ -290,6 +290,8 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase
{
void testShouldLoadUnprotectedPropertiesFromProtectedFile() throws
Exception {
// Arrange
File protectedFile = new
File("src/test/resources/conf/nifi_with_sensitive_properties_protected_aes.properties")
+
+ System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH,
protectedFile.path)
NiFiPropertiesLoader niFiPropertiesLoader =
NiFiPropertiesLoader.withKey(KEY_HEX)
final def EXPECTED_PLAIN_VALUES = [
@@ -378,7 +380,7 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase
{
logger.expected(msg)
// Assert
- assert msg == "Cannot read from bootstrap.conf"
+ assert msg =~ "Cannot read from .*bootstrap.conf"
}
@Test
@@ -399,7 +401,7 @@ class NiFiPropertiesLoaderGroovyTest extends GroovyTestCase
{
logger.expected(msg)
// Assert
- assert msg == "Cannot read from bootstrap.conf"
+ assert msg =~ "Cannot read from .*bootstrap.conf"
// Clean up to allow for indexing, etc.
Files.setPosixFilePermissions(unreadableFile.toPath(),
originalPermissions)
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
new file mode 100644
index 0000000..1d1a409
--- /dev/null
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# HTTP or HTTPS URI for HashiCorp Vault is required to enable the Sensitive
Properties Provider
+vault.uri=
+
+# Transit Path is required to enable the Sensitive Properties Provider
Protection Scheme 'hashicorp/vault/transit/{path}'
+vault.transit.path=
+
+# Token Authentication example properties
+# vault.authentication=TOKEN
+# vault.token=<token value>
+
+# Optional file supports authentication properties described in the Spring
Vault Environment Configuration
+#
https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration
+#
+# All authentication properties must be included in
bootstrap-hashicorp-vault.conf when this property is not specified.
+# Properties in bootstrap-hashicorp-vault.conf take precedence when the same
values are defined in both files.
+# Token Authentication is the default when the 'vault.authentication' property
is not specified.
+vault.authentication.properties.file=
+
+# Optional Timeout properties
+vault.connection.timeout=5 secs
+vault.read.timeout=15 secs
+
+# Optional TLS properties
+vault.ssl.enabledCipherSuites=
+vault.ssl.enabledProtocols=
+vault.ssl.key-store=
+vault.ssl.key-store-type=
+vault.ssl.key-store-password=
+vault.ssl.trust-store=
+vault.ssl.trust-store-type=
+vault.ssl.trust-store-password=
diff --git
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf
index 40486c0..778a699 100644
---
a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf
+++
b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/bootstrap.conf
@@ -58,6 +58,11 @@ java.arg.14=-Djava.awt.headless=true
# Root key in hexadecimal format for encrypted sensitive configuration values
nifi.bootstrap.sensitive.key=
+# Sensitive Property Provider configuration
+
+# HashiCorp Vault Sensitive Property Providers
+nifi.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
+
# Sets the provider of SecureRandom to /dev/urandom to prevent blocking on VMs
java.arg.15=-Djava.security.egd=file:/dev/urandom
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java
b/nifi-registry/nifi-registry-core/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java
index be9e282..a05c0f7 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java
+++
b/nifi-registry/nifi-registry-core/nifi-registry-properties/src/main/java/org/apache/nifi/registry/properties/NiFiRegistryPropertiesLoader.java
@@ -16,7 +16,7 @@
*/
package org.apache.nifi.registry.properties;
-import org.apache.nifi.properties.SensitivePropertyProtectionException;
+import org.apache.nifi.properties.BootstrapProperties;
import org.apache.nifi.properties.SensitivePropertyProvider;
import org.apache.nifi.properties.SensitivePropertyProviderFactory;
import org.apache.nifi.properties.StandardSensitivePropertyProviderFactory;
@@ -82,7 +82,8 @@ public class NiFiRegistryPropertiesLoader {
try {
return
NiFiRegistryBootstrapUtils.loadBootstrapProperties();
} catch (IOException e) {
- throw new
SensitivePropertyProtectionException("Could not load bootstrap.conf for
sensitive property provider configuration.", e);
+ logger.debug("Cannot read bootstrap.conf -- file
is missing or not readable. Defaulting to empty bootstrap.conf");
+ return BootstrapProperties.EMPTY;
}
});
}
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/util/NiFiRegistryBootstrapUtilsGroovyTest.groovy
b/nifi-registry/nifi-registry-core/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/util/NiFiRegistryBootstrapUtilsGroovyTest.groovy
index 465a122..c95bbb6 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/util/NiFiRegistryBootstrapUtilsGroovyTest.groovy
+++
b/nifi-registry/nifi-registry-core/nifi-registry-properties/src/test/groovy/org/apache/nifi/registry/properties/util/NiFiRegistryBootstrapUtilsGroovyTest.groovy
@@ -95,7 +95,7 @@ class NiFiRegistryBootstrapUtilsGroovyTest extends
GroovyTestCase {
logger.info(msg)
// Assert
- assert msg == "Cannot read from bootstrap.conf"
+ assert msg =~ "Cannot read from .*bootstrap.missing.conf"
}
@Test
@@ -114,7 +114,7 @@ class NiFiRegistryBootstrapUtilsGroovyTest extends
GroovyTestCase {
logger.info(msg)
// Assert
- assert msg == "Cannot read from bootstrap.conf"
+ assert msg =~ "Cannot read from
.*bootstrap.unreadable_file_permissions.conf"
} finally {
// Clean up to allow for indexing, etc.
Files.setPosixFilePermissions(unreadableFile.toPath(),
originalPermissions)
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
new file mode 100644
index 0000000..1d1a409
--- /dev/null
+++
b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap-hashicorp-vault.conf
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# HTTP or HTTPS URI for HashiCorp Vault is required to enable the Sensitive
Properties Provider
+vault.uri=
+
+# Transit Path is required to enable the Sensitive Properties Provider
Protection Scheme 'hashicorp/vault/transit/{path}'
+vault.transit.path=
+
+# Token Authentication example properties
+# vault.authentication=TOKEN
+# vault.token=<token value>
+
+# Optional file supports authentication properties described in the Spring
Vault Environment Configuration
+#
https://docs.spring.io/spring-vault/docs/2.3.x/reference/html/#vault.core.environment-vault-configuration
+#
+# All authentication properties must be included in
bootstrap-hashicorp-vault.conf when this property is not specified.
+# Properties in bootstrap-hashicorp-vault.conf take precedence when the same
values are defined in both files.
+# Token Authentication is the default when the 'vault.authentication' property
is not specified.
+vault.authentication.properties.file=
+
+# Optional Timeout properties
+vault.connection.timeout=5 secs
+vault.read.timeout=15 secs
+
+# Optional TLS properties
+vault.ssl.enabledCipherSuites=
+vault.ssl.enabledProtocols=
+vault.ssl.key-store=
+vault.ssl.key-store-type=
+vault.ssl.key-store-password=
+vault.ssl.trust-store=
+vault.ssl.trust-store-type=
+vault.ssl.trust-store-password=
diff --git
a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
index e4bf313..3663ba7 100644
---
a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
+++
b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/bootstrap.conf
@@ -51,4 +51,9 @@ java.arg.5=-Dsun.net.http.allowRestrictedHeaders=true
java.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol
# Master key in hexadecimal format for encrypted sensitive configuration values
-nifi.registry.bootstrap.sensitive.key=
\ No newline at end of file
+nifi.registry.bootstrap.sensitive.key=
+
+# Sensitive Property Provider configuration
+
+# HashiCorp Vault Sensitive Property Providers
+nifi.registry.bootstrap.protection.hashicorp.vault.conf=./conf/bootstrap-hashicorp-vault.conf
\ No newline at end of file
diff --git
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
index 9f48d6c..27cf192 100644
---
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
+++
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/properties/ConfigEncryptionTool.groovy
@@ -31,7 +31,6 @@ import org.apache.nifi.encrypt.PropertyEncryptor
import org.apache.nifi.encrypt.PropertyEncryptorFactory
import org.apache.nifi.flow.encryptor.FlowEncryptor
import org.apache.nifi.flow.encryptor.StandardFlowEncryptor
-import org.apache.nifi.registry.properties.util.NiFiRegistryBootstrapUtils
import org.apache.nifi.toolkit.tls.commandLine.CommandLineParseException
import org.apache.nifi.toolkit.tls.commandLine.ExitCode
import org.apache.nifi.util.NiFiBootstrapUtils
@@ -599,7 +598,8 @@ class ConfigEncryptionTool {
}
private NiFiPropertiesLoader getNiFiPropertiesLoader(final String keyHex) {
- return protectionScheme.requiresSecretKey() ?
NiFiPropertiesLoader.withKey(keyHex) : new NiFiPropertiesLoader()
+ return protectionScheme.requiresSecretKey() ||
migrationProtectionScheme.requiresSecretKey()
+ ? NiFiPropertiesLoader.withKey(keyHex) : new
NiFiPropertiesLoader()
}
/**
@@ -1566,7 +1566,7 @@ class ConfigEncryptionTool {
@Override
BootstrapProperties get() {
try {
-
NiFiRegistryBootstrapUtils.loadBootstrapProperties(bootstrapConfPath)
+
NiFiBootstrapUtils.loadBootstrapProperties(bootstrapConfPath)
} catch (final IOException e) {
throw new
SensitivePropertyProtectionException(e.getCause(), e)
}
diff --git
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy
index 919e2b0..b8433cf 100644
---
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy
+++
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryDecryptMode.groovy
@@ -17,7 +17,6 @@
package org.apache.nifi.toolkit.encryptconfig
import groovy.cli.commons.CliBuilder
-import org.apache.nifi.properties.ConfigEncryptionTool
import org.apache.nifi.properties.PropertyProtectionScheme
import org.apache.nifi.properties.StandardSensitivePropertyProviderFactory
import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil
@@ -117,7 +116,7 @@ class NiFiRegistryDecryptMode extends DecryptMode {
}
config.decryptionProvider =
StandardSensitivePropertyProviderFactory
- .withKeyAndBootstrapSupplier(config.key,
ConfigEncryptionTool.getBootstrapSupplier(config.inputBootstrapPath))
+ .withKeyAndBootstrapSupplier(config.key,
NiFiRegistryMode.getBootstrapSupplier(config.inputBootstrapPath))
.getProvider(config.protectionScheme)
run(config)
diff --git
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy
index 3b5bbce..ff36b22 100644
---
a/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy
+++
b/nifi-toolkit/nifi-toolkit-encrypt-config/src/main/groovy/org/apache/nifi/toolkit/encryptconfig/NiFiRegistryMode.groovy
@@ -20,10 +20,13 @@ import groovy.cli.commons.CliBuilder
import groovy.cli.commons.OptionAccessor
import org.apache.commons.cli.HelpFormatter
import org.apache.commons.cli.Options
+import org.apache.nifi.properties.BootstrapProperties
import org.apache.nifi.properties.ConfigEncryptionTool
import org.apache.nifi.properties.PropertyProtectionScheme
+import org.apache.nifi.properties.SensitivePropertyProtectionException
import org.apache.nifi.properties.SensitivePropertyProvider
import org.apache.nifi.properties.StandardSensitivePropertyProviderFactory
+import org.apache.nifi.registry.properties.util.NiFiRegistryBootstrapUtils
import org.apache.nifi.toolkit.encryptconfig.util.BootstrapUtil
import
org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryAuthorizersXmlEncryptor
import
org.apache.nifi.toolkit.encryptconfig.util.NiFiRegistryIdentityProvidersXmlEncryptor
@@ -33,6 +36,8 @@ import org.apache.nifi.util.console.TextDevices
import org.slf4j.Logger
import org.slf4j.LoggerFactory
+import java.util.function.Supplier
+
class NiFiRegistryMode implements ToolMode {
private static final Logger logger =
LoggerFactory.getLogger(NiFiRegistryMode.class)
@@ -45,6 +50,19 @@ class NiFiRegistryMode implements ToolMode {
verboseEnabled = false
}
+ static Supplier<BootstrapProperties> getBootstrapSupplier(final String
bootstrapConfPath) {
+ new Supplier<BootstrapProperties>() {
+ @Override
+ BootstrapProperties get() {
+ try {
+
NiFiRegistryBootstrapUtils.loadBootstrapProperties(bootstrapConfPath)
+ } catch (final IOException e) {
+ throw new
SensitivePropertyProtectionException(e.getCause(), e)
+ }
+ }
+ }
+ }
+
@Override
void run(String[] args) {
try {
@@ -318,12 +336,12 @@ class NiFiRegistryMode implements ToolMode {
throw new RuntimeException("Failed to configure tool, could
not determine encryption key. Must provide -p, -k, or -b. If using -b,
bootstrap.conf argument must already contain root key.")
}
encryptionProvider = StandardSensitivePropertyProviderFactory
- .withKeyAndBootstrapSupplier(encryptionKey,
ConfigEncryptionTool.getBootstrapSupplier(inputBootstrapPath))
- .getProvider(oldProtectionScheme)
+ .withKeyAndBootstrapSupplier(encryptionKey,
getBootstrapSupplier(inputBootstrapPath))
+ .getProvider(protectionScheme)
decryptionProvider = decryptionKey ?
StandardSensitivePropertyProviderFactory
- .withKeyAndBootstrapSupplier(decryptionKey,
ConfigEncryptionTool.getBootstrapSupplier(inputBootstrapPath))
- .getProvider(protectionScheme) : null
+ .withKeyAndBootstrapSupplier(decryptionKey,
getBootstrapSupplier(inputBootstrapPath))
+ .getProvider(oldProtectionScheme) : null
if (handlingNiFiRegistryProperties) {
propertiesEncryptor = new
NiFiRegistryPropertiesEncryptor(encryptionProvider, decryptionProvider)
