This is an automated email from the ASF dual-hosted git repository.
exceptionfactory pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
The following commit(s) were added to refs/heads/main by this push:
new 115bba9 NIFI-8523 Added SFTP algorithm and cipher properties
115bba9 is described below
commit 115bba9ab01d432da60d1fc29359812690a5b453
Author: Jon Kessler <[email protected]>
AuthorDate: Thu May 6 16:10:36 2021 +0000
NIFI-8523 Added SFTP algorithm and cipher properties
- Updated secure FTP processors to configure which algorithms, ciphers and
message authentication codes are allowed to be used by the SSH Client
- Included Expression Language Variable Registry support for properties
This closes #5061
Signed-off-by: David Handermann <[email protected]>
---
.../apache/nifi/processors/standard/FetchSFTP.java | 4 +
.../apache/nifi/processors/standard/GetSFTP.java | 4 +
.../apache/nifi/processors/standard/ListSFTP.java | 4 +
.../apache/nifi/processors/standard/PutSFTP.java | 4 +
.../processors/standard/util/SFTPTransfer.java | 113 +++++++++++++++++++++
.../processors/standard/util/TestSFTPTransfer.java | 38 +++++++
6 files changed, 167 insertions(+)
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FetchSFTP.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FetchSFTP.java
index 2124b68..69281e5 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FetchSFTP.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/FetchSFTP.java
@@ -87,6 +87,10 @@ public class FetchSFTP extends FetchFileTransfer {
properties.add(FTPTransfer.HTTP_PROXY_USERNAME);
properties.add(FTPTransfer.HTTP_PROXY_PASSWORD);
properties.add(FILE_NOT_FOUND_LOG_LEVEL);
+ properties.add(SFTPTransfer.CIPHERS_ALLOWED);
+ properties.add(SFTPTransfer.KEY_ALGORITHMS_ALLOWED);
+ properties.add(SFTPTransfer.KEY_EXCHANGE_ALGORITHMS_ALLOWED);
+ properties.add(SFTPTransfer.MESSAGE_AUTHENTICATION_CODES_ALLOWED);
return properties;
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
index 5a3d667..0e105a8 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetSFTP.java
@@ -89,6 +89,10 @@ public class GetSFTP extends GetFileTransfer {
properties.add(FTPTransfer.PROXY_PORT);
properties.add(FTPTransfer.HTTP_PROXY_USERNAME);
properties.add(FTPTransfer.HTTP_PROXY_PASSWORD);
+ properties.add(SFTPTransfer.CIPHERS_ALLOWED);
+ properties.add(SFTPTransfer.KEY_ALGORITHMS_ALLOWED);
+ properties.add(SFTPTransfer.KEY_EXCHANGE_ALGORITHMS_ALLOWED);
+ properties.add(SFTPTransfer.MESSAGE_AUTHENTICATION_CODES_ALLOWED);
this.properties = Collections.unmodifiableList(properties);
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
index c345b3b..704cebc 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/ListSFTP.java
@@ -116,6 +116,10 @@ public class ListSFTP extends ListFileTransfer {
properties.add(ListFile.MAX_AGE);
properties.add(ListFile.MIN_SIZE);
properties.add(ListFile.MAX_SIZE);
+ properties.add(SFTPTransfer.CIPHERS_ALLOWED);
+ properties.add(SFTPTransfer.KEY_ALGORITHMS_ALLOWED);
+ properties.add(SFTPTransfer.KEY_EXCHANGE_ALGORITHMS_ALLOWED);
+ properties.add(SFTPTransfer.MESSAGE_AUTHENTICATION_CODES_ALLOWED);
return properties;
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSFTP.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSFTP.java
index 35bb174..e431042 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSFTP.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/PutSFTP.java
@@ -80,6 +80,10 @@ public class PutSFTP extends PutFileTransfer<SFTPTransfer> {
properties.add(FTPTransfer.PROXY_PORT);
properties.add(FTPTransfer.HTTP_PROXY_USERNAME);
properties.add(FTPTransfer.HTTP_PROXY_PASSWORD);
+ properties.add(SFTPTransfer.CIPHERS_ALLOWED);
+ properties.add(SFTPTransfer.KEY_ALGORITHMS_ALLOWED);
+ properties.add(SFTPTransfer.KEY_EXCHANGE_ALGORITHMS_ALLOWED);
+ properties.add(SFTPTransfer.MESSAGE_AUTHENTICATION_CODES_ALLOWED);
this.properties = Collections.unmodifiableList(properties);
}
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
index e7980cd..01cc6cf 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/SFTPTransfer.java
@@ -18,8 +18,10 @@ package org.apache.nifi.processors.standard.util;
import net.schmizz.keepalive.KeepAlive;
import net.schmizz.keepalive.KeepAliveProvider;
+import net.schmizz.sshj.Config;
import net.schmizz.sshj.DefaultConfig;
import net.schmizz.sshj.SSHClient;
+import net.schmizz.sshj.common.Factory;
import net.schmizz.sshj.connection.ConnectionException;
import net.schmizz.sshj.connection.ConnectionImpl;
import net.schmizz.sshj.sftp.FileAttributes;
@@ -72,17 +74,50 @@ import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
import static
org.apache.nifi.processors.standard.util.FTPTransfer.createComponentProxyConfigSupplier;
public class SFTPTransfer implements FileTransfer {
+ private static final Set<String> DEFAULT_KEY_ALGORITHM_NAMES;
+ private static final Set<String> DEFAULT_CIPHER_NAMES;
+ private static final Set<String> DEFAULT_MESSAGE_AUTHENTICATION_CODE_NAMES;
+ private static final Set<String> DEFAULT_KEY_EXCHANGE_ALGORITHM_NAMES;
+
+ static {
+ DefaultConfig defaultConfig = new DefaultConfig();
+
+ DEFAULT_KEY_ALGORITHM_NAMES =
Collections.unmodifiableSet(defaultConfig.getKeyAlgorithms().stream()
+ .map(Factory.Named::getName).collect(Collectors.toSet()));
+ DEFAULT_CIPHER_NAMES =
Collections.unmodifiableSet(defaultConfig.getCipherFactories().stream()
+ .map(Factory.Named::getName).collect(Collectors.toSet()));
+ DEFAULT_MESSAGE_AUTHENTICATION_CODE_NAMES =
Collections.unmodifiableSet(defaultConfig.getMACFactories().stream()
+ .map(Factory.Named::getName).collect(Collectors.toSet()));
+ DEFAULT_KEY_EXCHANGE_ALGORITHM_NAMES =
Collections.unmodifiableSet(defaultConfig.getKeyExchangeFactories().stream()
+ .map(Factory.Named::getName).collect(Collectors.toSet()));
+ }
+
+ /**
+ * Converts a set of names into an alphabetically ordered comma separated
value list.
+ *
+ * @param factorySetNames The set of names
+ * @return An alphabetically ordered comma separated value list of names
+ */
+ private static String convertFactorySetToString(Set<String>
factorySetNames) {
+ return factorySetNames
+ .stream()
+ .sorted()
+ .collect(Collectors.joining(", "));
+ }
public static final PropertyDescriptor PRIVATE_KEY_PATH = new
PropertyDescriptor.Builder()
.name("Private Key Path")
@@ -131,6 +166,44 @@ public class SFTPTransfer implements FileTransfer {
.required(true)
.build();
+ public static final PropertyDescriptor KEY_ALGORITHMS_ALLOWED = new
PropertyDescriptor.Builder()
+ .name("Key Algorithms Allowed")
+ .displayName("Key Algorithms Allowed")
+ .description("A comma-separated list of Key Algorithms allowed for
SFTP connections. Leave unset to allow all. Available options are: "
+ + convertFactorySetToString(DEFAULT_KEY_ALGORITHM_NAMES))
+ .required(false)
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CIPHERS_ALLOWED = new
PropertyDescriptor.Builder()
+ .name("Ciphers Allowed")
+ .displayName("Ciphers Allowed")
+ .description("A comma-separated list of Ciphers allowed for SFTP
connections. Leave unset to allow all. Available options are: " +
convertFactorySetToString(DEFAULT_CIPHER_NAMES))
+ .required(false)
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor
MESSAGE_AUTHENTICATION_CODES_ALLOWED = new PropertyDescriptor.Builder()
+ .name("Message Authentication Codes Allowed")
+ .displayName("Message Authentication Codes Allowed")
+ .description("A comma-separated list of Message Authentication
Codes allowed for SFTP connections. Leave unset to allow all. Available options
are: "
+ +
convertFactorySetToString(DEFAULT_MESSAGE_AUTHENTICATION_CODE_NAMES))
+ .required(false)
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor KEY_EXCHANGE_ALGORITHMS_ALLOWED =
new PropertyDescriptor.Builder()
+ .name("Key Exchange Algorithms Allowed")
+ .displayName("Key Exchange Algorithms Allowed")
+ .description("A comma-separated list of Key Exchange Algorithms
allowed for SFTP connections. Leave unset to allow all. Available options are: "
+ +
convertFactorySetToString(DEFAULT_KEY_EXCHANGE_ALGORITHM_NAMES))
+ .required(false)
+
.expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
+ .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
+ .build();
/**
* Property which is used to decide if the {@link
#ensureDirectoryExists(FlowFile, File)} method should perform a {@link
SFTPClient#ls(String)} before calling
@@ -521,6 +594,8 @@ public class SFTPTransfer implements FileTransfer {
sshClientConfig.setKeepAliveProvider(NO_OP_KEEP_ALIVE);
}
+ updateConfigAlgorithms(sshClientConfig);
+
final SSHClient sshClient = new SSHClient(sshClientConfig);
// Create a Proxy if the config was specified, proxy will be null if
type was NO_PROXY
@@ -642,6 +717,44 @@ public class SFTPTransfer implements FileTransfer {
return sftpClient;
}
+ void updateConfigAlgorithms(final Config config) {
+ if (ctx.getProperty(CIPHERS_ALLOWED).isSet()) {
+ Set<String> allowedCiphers =
Arrays.stream(ctx.getProperty(CIPHERS_ALLOWED).evaluateAttributeExpressions().getValue().split(","))
+ .map(String::trim)
+ .collect(Collectors.toSet());
+ config.setCipherFactories(config.getCipherFactories().stream()
+ .filter(cipherNamed ->
allowedCiphers.contains(cipherNamed.getName()))
+ .collect(Collectors.toList()));
+ }
+
+ if (ctx.getProperty(KEY_ALGORITHMS_ALLOWED).isSet()) {
+ Set<String> allowedKeyAlgorithms =
Arrays.stream(ctx.getProperty(KEY_ALGORITHMS_ALLOWED).evaluateAttributeExpressions().getValue().split(","))
+ .map(String::trim)
+ .collect(Collectors.toSet());
+ config.setKeyAlgorithms(config.getKeyAlgorithms().stream()
+ .filter(keyAlgorithmNamed ->
allowedKeyAlgorithms.contains(keyAlgorithmNamed.getName()))
+ .collect(Collectors.toList()));
+ }
+
+ if (ctx.getProperty(KEY_EXCHANGE_ALGORITHMS_ALLOWED).isSet()) {
+ Set<String> allowedKeyExchangeAlgorithms =
Arrays.stream(ctx.getProperty(KEY_EXCHANGE_ALGORITHMS_ALLOWED).evaluateAttributeExpressions().getValue().split(","))
+ .map(String::trim)
+ .collect(Collectors.toSet());
+
config.setKeyExchangeFactories(config.getKeyExchangeFactories().stream()
+ .filter(keyExchangeNamed ->
allowedKeyExchangeAlgorithms.contains(keyExchangeNamed.getName()))
+ .collect(Collectors.toList()));
+ }
+
+ if (ctx.getProperty(MESSAGE_AUTHENTICATION_CODES_ALLOWED).isSet()) {
+ Set<String> allowedMessageAuthenticationCodes =
Arrays.stream(ctx.getProperty(MESSAGE_AUTHENTICATION_CODES_ALLOWED).evaluateAttributeExpressions().getValue().split(","))
+ .map(String::trim)
+ .collect(Collectors.toSet());
+ config.setMACFactories(config.getMACFactories().stream()
+ .filter(macNamed ->
allowedMessageAuthenticationCodes.contains(macNamed.getName()))
+ .collect(Collectors.toList()));
+ }
+ }
+
@Override
public String getHomeDirectory(final FlowFile flowFile) throws IOException
{
getSFTPClient(flowFile);
diff --git
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestSFTPTransfer.java
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestSFTPTransfer.java
index e9d9a0b..e4307aa 100644
---
a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestSFTPTransfer.java
+++
b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/util/TestSFTPTransfer.java
@@ -16,13 +16,18 @@
*/
package org.apache.nifi.processors.standard.util;
+import net.schmizz.sshj.DefaultConfig;
+import net.schmizz.sshj.common.Factory;
import net.schmizz.sshj.sftp.Response;
import net.schmizz.sshj.sftp.SFTPClient;
import net.schmizz.sshj.sftp.SFTPException;
+import org.apache.nifi.components.PropertyDescriptor;
import org.apache.nifi.flowfile.FlowFile;
import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.mock.MockComponentLogger;
import org.apache.nifi.processor.ProcessContext;
import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.MockPropertyContext;
import org.apache.nifi.util.MockPropertyValue;
import org.junit.Test;
import org.slf4j.Logger;
@@ -30,7 +35,10 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -244,4 +252,34 @@ public class TestSFTPTransfer {
verify(sftpClient).mkdir(eq("/dir1/dir2/dir3")); // dir3 was created
blindly.
}
+ @Test
+ public void testRestrictSSHOptions() {
+ Map<PropertyDescriptor, String> propertyDescriptorValues = new
HashMap<>();
+
+ DefaultConfig defaultConfig = new DefaultConfig();
+
+ String allowedMac =
defaultConfig.getMACFactories().stream().map(Factory.Named::getName).collect(Collectors.toList()).get(0);
+ String allowedKeyAlgorithm =
defaultConfig.getKeyAlgorithms().stream().map(Factory.Named::getName).collect(Collectors.toList()).get(0);
+ String allowedKeyExchangeAlgorithm =
defaultConfig.getKeyExchangeFactories().stream().map(Factory.Named::getName).collect(Collectors.toList()).get(0);
+ String allowedCipher =
defaultConfig.getCipherFactories().stream().map(Factory.Named::getName).collect(Collectors.toList()).get(0);
+
+
propertyDescriptorValues.put(SFTPTransfer.MESSAGE_AUTHENTICATION_CODES_ALLOWED,
allowedMac);
+ propertyDescriptorValues.put(SFTPTransfer.CIPHERS_ALLOWED,
allowedCipher);
+ propertyDescriptorValues.put(SFTPTransfer.KEY_ALGORITHMS_ALLOWED,
allowedKeyAlgorithm);
+
propertyDescriptorValues.put(SFTPTransfer.KEY_EXCHANGE_ALGORITHMS_ALLOWED,
allowedKeyExchangeAlgorithm);
+ MockPropertyContext mockPropertyContext = new
MockPropertyContext(propertyDescriptorValues);
+ SFTPTransfer sftpTransfer = new SFTPTransfer(mockPropertyContext, new
MockComponentLogger());
+
+ sftpTransfer.updateConfigAlgorithms(defaultConfig);
+
+ assertEquals(1, defaultConfig.getCipherFactories().size());
+ assertEquals(1, defaultConfig.getKeyAlgorithms().size());
+ assertEquals(1, defaultConfig.getKeyExchangeFactories().size());
+ assertEquals(1, defaultConfig.getMACFactories().size());
+
+ assertEquals(allowedCipher,
defaultConfig.getCipherFactories().get(0).getName());
+ assertEquals(allowedKeyAlgorithm,
defaultConfig.getKeyAlgorithms().get(0).getName());
+ assertEquals(allowedKeyExchangeAlgorithm,
defaultConfig.getKeyExchangeFactories().get(0).getName());
+ assertEquals(allowedMac,
defaultConfig.getMACFactories().get(0).getName());
+ }
}
