This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 07c4a05 NIFI-9412: Autogenerate sensitive key in MiNiFi if not present
07c4a05 is described below
commit 07c4a05cdf432689feb12ebb150552aa44d41f1d
Author: Matthew Burgess <[email protected]>
AuthorDate: Mon Nov 29 20:54:54 2021 -0500
NIFI-9412: Autogenerate sensitive key in MiNiFi if not present
This closes #5558
Signed-off-by: David Handermann <[email protected]>
---
minifi/minifi-bootstrap/pom.xml | 5 +++
.../minifi/bootstrap/util/ConfigTransformer.java | 52 +++++++++++++---------
.../bootstrap/util/ConfigTransformerTest.java | 33 +++++---------
.../resources/MINIFI-216/nifi.properties.before | 3 +-
.../resources/MINIFI-245/nifi.properties.before | 3 +-
.../src/test/resources/MINIFI-277/nifi.properties | 4 +-
.../resources/NIFI-8753/nifi.properties.before | 3 +-
7 files changed, 53 insertions(+), 50 deletions(-)
diff --git a/minifi/minifi-bootstrap/pom.xml b/minifi/minifi-bootstrap/pom.xml
index 787f2b7..379e407 100644
--- a/minifi/minifi-bootstrap/pom.xml
+++ b/minifi/minifi-bootstrap/pom.xml
@@ -61,6 +61,11 @@ limitations under the License.
<artifactId>nifi-utils</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-properties</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
diff --git
a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java
b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java
index cc14dca..10458be 100644
---
a/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java
+++
b/minifi/minifi-bootstrap/src/main/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformer.java
@@ -17,7 +17,6 @@
package org.apache.nifi.minifi.bootstrap.util;
-
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.commons.lang3.StringUtils;
import
org.apache.nifi.minifi.bootstrap.configuration.ConfigurationChangeException;
@@ -46,6 +45,7 @@ import
org.apache.nifi.minifi.commons.schema.common.ConvertableSchema;
import org.apache.nifi.minifi.commons.schema.common.Schema;
import org.apache.nifi.minifi.commons.schema.common.StringUtil;
import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader;
+import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.DOMException;
@@ -69,7 +69,9 @@ import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.security.SecureRandom;
import java.util.Arrays;
+import java.util.Base64;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -80,9 +82,10 @@ import java.util.zip.GZIPOutputStream;
public final class ConfigTransformer {
// Underlying version of NIFI will be using
- public static final String NIFI_VERSION = "1.8.0";
public static final String ROOT_GROUP = "Root-Group";
- public static final String NIFI_VERSION_KEY = "nifi.version";
+
+ private static final Base64.Encoder KEY_ENCODER =
Base64.getEncoder().withoutPadding();
+ private static final int SENSITIVE_PROPERTIES_KEY_LENGTH = 24;
public static final Logger logger =
LoggerFactory.getLogger(ConfigTransformer.class);
@@ -178,8 +181,7 @@ public final class ConfigTransformer {
ProvenanceRepositorySchema provenanceRepositorySchema =
configSchema.getProvenanceRepositorySchema();
OrderedProperties orderedProperties = new OrderedProperties();
- orderedProperties.setProperty(NIFI_VERSION_KEY, NIFI_VERSION, "#
Core Properties #" + System.lineSeparator());
- orderedProperties.setProperty("nifi.flow.configuration.file",
"./conf/flow.xml.gz");
+ orderedProperties.setProperty("nifi.flow.configuration.file",
"./conf/flow.xml.gz", "# Core Properties #" + System.lineSeparator());
orderedProperties.setProperty("nifi.flow.configuration.archive.enabled",
"false");
orderedProperties.setProperty("nifi.flow.configuration.archive.dir",
"./conf/archive/");
orderedProperties.setProperty("nifi.flowcontroller.autoResumeState", "true");
@@ -258,7 +260,17 @@ public final class ConfigTransformer {
orderedProperties.setProperty("nifi.web.jetty.threads", "200");
final String sensitivePropertiesKey = sensitiveProperties.getKey();
- final String notnullSensitivePropertiesKey =
sensitivePropertiesKey != null ? sensitivePropertiesKey : "";
+ final String notnullSensitivePropertiesKey;
+ // Auto-generate the sensitive properties key if not provided,
NiFi security libraries require it
+ if (StringUtil.isNullOrEmpty(sensitivePropertiesKey)) {
+ logger.warn("Generating Random Sensitive Properties Key [{}]",
NiFiProperties.SENSITIVE_PROPS_KEY);
+ final SecureRandom secureRandom = new SecureRandom();
+ final byte[] sensitivePropertiesKeyBinary = new
byte[SENSITIVE_PROPERTIES_KEY_LENGTH];
+ secureRandom.nextBytes(sensitivePropertiesKeyBinary);
+ notnullSensitivePropertiesKey =
KEY_ENCODER.encodeToString(sensitivePropertiesKeyBinary);
+ } else {
+ notnullSensitivePropertiesKey = sensitivePropertiesKey;
+ }
orderedProperties.setProperty("nifi.sensitive.props.key",
notnullSensitivePropertiesKey, System.lineSeparator() + "# security properties
#");
orderedProperties.setProperty("nifi.sensitive.props.algorithm",
sensitiveProperties.getAlgorithm());
@@ -719,19 +731,19 @@ public final class ConfigTransformer {
public static final String PROPERTIES_FILE_APACHE_2_0_LICENSE =
" Licensed to the Apache Software Foundation (ASF) under one or
more\n" +
- "# contributor license agreements. See the NOTICE file
distributed with\n" +
- "# this work for additional information regarding copyright
ownership.\n" +
- "# The ASF licenses this file to You under the Apache License,
Version 2.0\n" +
- "# (the \"License\"); you may not use this file except in
compliance with\n" +
- "# the License. You may obtain a copy of the License at\n" +
- "#\n" +
- "# http://www.apache.org/licenses/LICENSE-2.0\n"; +
- "#\n" +
- "# Unless required by applicable law or agreed to in writing,
software\n" +
- "# distributed under the License is distributed on an \"AS IS\"
BASIS,\n" +
- "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.\n" +
- "# See the License for the specific language governing permissions
and\n" +
- "# limitations under the License.\n" +
- "\n";
+ "# contributor license agreements. See the NOTICE file
distributed with\n" +
+ "# this work for additional information regarding
copyright ownership.\n" +
+ "# The ASF licenses this file to You under the Apache
License, Version 2.0\n" +
+ "# (the \"License\"); you may not use this file except in
compliance with\n" +
+ "# the License. You may obtain a copy of the License
at\n" +
+ "#\n" +
+ "# http://www.apache.org/licenses/LICENSE-2.0\n"; +
+ "#\n" +
+ "# Unless required by applicable law or agreed to in
writing, software\n" +
+ "# distributed under the License is distributed on an \"AS
IS\" BASIS,\n" +
+ "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
express or implied.\n" +
+ "# See the License for the specific language governing
permissions and\n" +
+ "# limitations under the License.\n" +
+ "\n";
}
diff --git
a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java
b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java
index 7c621cb..bb01784 100644
---
a/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java
+++
b/minifi/minifi-bootstrap/src/test/java/org/apache/nifi/minifi/bootstrap/util/ConfigTransformerTest.java
@@ -33,6 +33,7 @@ import org.apache.nifi.minifi.commons.schema.ReportingSchema;
import org.apache.nifi.minifi.commons.schema.common.StringUtil;
import org.apache.nifi.minifi.commons.schema.exception.SchemaLoaderException;
import org.apache.nifi.minifi.commons.schema.serialization.SchemaLoader;
+import org.apache.nifi.util.StringUtils;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -78,7 +79,6 @@ public class ConfigTransformerTest {
public static final Map<String, Integer> PG_ELEMENT_ORDER_MAP =
generateOrderMap(
Arrays.asList("processor", "inputPort", "outputPort", "funnel",
"processGroup", "remoteProcessGroup", "connection"));
private XPathFactory xPathFactory;
- private Document document;
private Element config;
private DocumentBuilder documentBuilder;
@@ -88,7 +88,7 @@ public class ConfigTransformerTest {
@Before
public void setup() throws ParserConfigurationException {
documentBuilder =
DocumentBuilderFactory.newInstance().newDocumentBuilder();
- document = documentBuilder.newDocument();
+ final Document document = documentBuilder.newDocument();
config = document.createElement("config");
xPathFactory = XPathFactory.newInstance();
}
@@ -169,8 +169,6 @@ public class ConfigTransformerTest {
try (InputStream pre216PropertiesStream =
ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/nifi.properties.before"))
{
pre216Properties.load(pre216PropertiesStream);
}
- pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY,
ConfigTransformer.NIFI_VERSION);
-
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (InputStream configStream =
ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/config.yml"))
{
ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream),
outputStream);
@@ -189,8 +187,6 @@ public class ConfigTransformerTest {
try (InputStream pre216PropertiesStream =
ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/nifi.properties.before"))
{
pre216Properties.load(pre216PropertiesStream);
}
- pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY,
ConfigTransformer.NIFI_VERSION);
-
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (InputStream configStream =
ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-216/configOverrides.yml"))
{
ConfigSchema configSchema =
SchemaLoader.loadConfigSchemaFromYaml(configStream);
@@ -214,8 +210,6 @@ public class ConfigTransformerTest {
try (InputStream pre216PropertiesStream =
ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-277/nifi.properties"))
{
initialProperties.load(pre216PropertiesStream);
}
- initialProperties.setProperty(ConfigTransformer.NIFI_VERSION_KEY,
ConfigTransformer.NIFI_VERSION);
-
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (InputStream configStream =
ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-277/config.yml"))
{
ConfigSchema configSchema =
SchemaLoader.loadConfigSchemaFromYaml(configStream);
@@ -421,7 +415,7 @@ public class ConfigTransformerTest {
File inputFile = new
File("./src/test/resources/config-invalid.yml");
ConfigTransformer.transformConfigFile(new
FileInputStream(inputFile), "./target/", null);
fail("Invalid configuration file was not detected.");
- } catch (SchemaLoaderException e){
+ } catch (SchemaLoaderException e) {
assertEquals("Provided YAML configuration is not a Map",
e.getMessage());
}
}
@@ -432,7 +426,7 @@ public class ConfigTransformerTest {
File inputFile = new
File("./src/test/resources/config-malformed-field.yml");
ConfigTransformer.transformConfigFile(new
FileInputStream(inputFile), "./target/", null);
fail("Invalid configuration file was not detected.");
- } catch (InvalidConfigurationException e){
+ } catch (InvalidConfigurationException e) {
assertEquals("Failed to transform config file due to:['threshold'
in section 'Swap' because it is found but could not be parsed as a Number]",
e.getMessage());
}
}
@@ -443,7 +437,7 @@ public class ConfigTransformerTest {
File inputFile = new File("./src/test/resources/config-empty.yml");
ConfigTransformer.transformConfigFile(new
FileInputStream(inputFile), "./target/", null);
fail("Invalid configuration file was not detected.");
- } catch (SchemaLoaderException e){
+ } catch (SchemaLoaderException e) {
assertEquals("Provided YAML configuration is not a Map",
e.getMessage());
}
}
@@ -454,7 +448,7 @@ public class ConfigTransformerTest {
File inputFile = new
File("./src/test/resources/config-missing-required-field.yml");
ConfigTransformer.transformConfigFile(new
FileInputStream(inputFile), "./target/", null);
fail("Invalid configuration file was not detected.");
- } catch (InvalidConfigurationException e){
+ } catch (InvalidConfigurationException e) {
assertEquals("Failed to transform config file due to:['class' in
section 'Processors' because it was not found and it is required]",
e.getMessage());
}
}
@@ -465,7 +459,7 @@ public class ConfigTransformerTest {
File inputFile = new
File("./src/test/resources/config-multiple-problems.yml");
ConfigTransformer.transformConfigFile(new
FileInputStream(inputFile), "./target/", null);
fail("Invalid configuration file was not detected.");
- } catch (InvalidConfigurationException e){
+ } catch (InvalidConfigurationException e) {
assertEquals("Failed to transform config file due to:['class' in
section 'Processors' because it was not found and it is required], " +
"['scheduling strategy' in section 'Provenance Reporting'
because it is not a valid scheduling strategy], " +
"['source name' in section 'Connections' because it was
not found and it is required]", e.getMessage());
@@ -699,8 +693,6 @@ public class ConfigTransformerTest {
try (InputStream pre216PropertiesStream =
ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-245/nifi.properties.before"))
{
pre216Properties.load(pre216PropertiesStream);
}
- pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY,
ConfigTransformer.NIFI_VERSION);
-
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (InputStream configStream =
ConfigTransformerTest.class.getClassLoader().getResourceAsStream("MINIFI-245/config.yml"))
{
ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream),
outputStream);
@@ -710,7 +702,7 @@ public class ConfigTransformerTest {
for (String name : pre216Properties.stringPropertyNames()) {
// Verify the Content Repo property was overridden
- if("nifi.content.repository.implementation".equals(name)) {
+ if ("nifi.content.repository.implementation".equals(name)) {
assertNotEquals("Property key " + name + " was not
overridden.", pre216Properties.getProperty(name), properties.getProperty(name));
} else {
assertEquals("Property key " + name + " doesn't match.",
pre216Properties.getProperty(name), properties.getProperty(name));
@@ -724,8 +716,6 @@ public class ConfigTransformerTest {
try (InputStream pre216PropertiesStream =
ConfigTransformerTest.class.getClassLoader().getResourceAsStream("NIFI-8753/nifi.properties.before"))
{
pre216Properties.load(pre216PropertiesStream);
}
- pre216Properties.setProperty(ConfigTransformer.NIFI_VERSION_KEY,
ConfigTransformer.NIFI_VERSION);
-
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (InputStream configStream =
ConfigTransformerTest.class.getClassLoader().getResourceAsStream("NIFI-8753/config.yml"))
{
ConfigTransformer.writeNiFiProperties(SchemaLoader.loadConfigSchemaFromYaml(configStream),
outputStream);
@@ -735,7 +725,7 @@ public class ConfigTransformerTest {
for (String name : pre216Properties.stringPropertyNames()) {
// Verify the Content Repo property was overridden
- if("nifi.flowfile.repository.implementation".equals(name)) {
+ if ("nifi.flowfile.repository.implementation".equals(name)) {
assertNotEquals("Property key " + name + " was not
overridden.", pre216Properties.getProperty(name), properties.getProperty(name));
} else {
assertEquals("Property key " + name + " doesn't match.",
pre216Properties.getProperty(name), properties.getProperty(name));
@@ -751,7 +741,8 @@ public class ConfigTransformerTest {
}
final Properties properties = new Properties();
properties.load(new ByteArrayInputStream(outputStream.toByteArray()));
- assertEquals("", properties.getProperty("nifi.sensitive.props.key"));
+ // The property should not be empty/null as it is auto-generated when
missing
+
assertTrue(StringUtils.isNotEmpty(properties.getProperty("nifi.sensitive.props.key")));
}
private String getText(Element element, String path) throws
XPathExpressionException {
@@ -780,7 +771,7 @@ public class ConfigTransformerTest {
if (index != null) {
if (elementOrderList > index) {
fail("Found " + nodeName + " after " +
lastOrderedElementName + "; expected all " + nodeName + " elements to come
before the following elements: " + orderMap.entrySet().stream()
- .filter(e -> e.getValue() > index
).sorted(Comparator.comparingInt(e -> e.getValue())).map(e ->
e.getKey()).collect(Collectors.joining(", ")));
+ .filter(e -> e.getValue() >
index).sorted(Comparator.comparingInt(e -> e.getValue())).map(e ->
e.getKey()).collect(Collectors.joining(", ")));
}
lastOrderedElementName = nodeName;
elementOrderList = index;
diff --git
a/minifi/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before
b/minifi/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before
index 2de0115..0c60b4b 100644
---
a/minifi/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before
+++
b/minifi/minifi-bootstrap/src/test/resources/MINIFI-216/nifi.properties.before
@@ -15,7 +15,6 @@
# Core Properties #
-nifi.version=1.1.0
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
@@ -92,7 +91,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
# security properties #
-nifi.sensitive.props.key=
+# This needs to be ignored during unit testing: nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.security.keystore=/tmp/ssl/localhost-ks.jks
diff --git
a/minifi/minifi-bootstrap/src/test/resources/MINIFI-245/nifi.properties.before
b/minifi/minifi-bootstrap/src/test/resources/MINIFI-245/nifi.properties.before
index 2de0115..0c60b4b 100644
---
a/minifi/minifi-bootstrap/src/test/resources/MINIFI-245/nifi.properties.before
+++
b/minifi/minifi-bootstrap/src/test/resources/MINIFI-245/nifi.properties.before
@@ -15,7 +15,6 @@
# Core Properties #
-nifi.version=1.1.0
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
@@ -92,7 +91,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
# security properties #
-nifi.sensitive.props.key=
+# This needs to be ignored during unit testing: nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.security.keystore=/tmp/ssl/localhost-ks.jks
diff --git
a/minifi/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties
b/minifi/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties
index 9842143..5b2b1a7 100644
--- a/minifi/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties
+++ b/minifi/minifi-bootstrap/src/test/resources/MINIFI-277/nifi.properties
@@ -15,8 +15,6 @@
# Core Properties #
-nifi.version=1.1.0
-
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
@@ -94,7 +92,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
# security properties #
-nifi.sensitive.props.key=
+# This needs to be ignored during unit testing: nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.security.keystore=/tmp/ssl/localhost-ks.jks
diff --git
a/minifi/minifi-bootstrap/src/test/resources/NIFI-8753/nifi.properties.before
b/minifi/minifi-bootstrap/src/test/resources/NIFI-8753/nifi.properties.before
index 2de0115..0c60b4b 100644
---
a/minifi/minifi-bootstrap/src/test/resources/NIFI-8753/nifi.properties.before
+++
b/minifi/minifi-bootstrap/src/test/resources/NIFI-8753/nifi.properties.before
@@ -15,7 +15,6 @@
# Core Properties #
-nifi.version=1.1.0
nifi.flow.configuration.file=./conf/flow.xml.gz
nifi.flow.configuration.archive.enabled=false
nifi.flow.configuration.archive.dir=./conf/archive/
@@ -92,7 +91,7 @@ nifi.web.jetty.working.directory=./work/jetty
nifi.web.jetty.threads=200
# security properties #
-nifi.sensitive.props.key=
+# This needs to be ignored during unit testing: nifi.sensitive.props.key=
nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL
nifi.security.keystore=/tmp/ssl/localhost-ks.jks
