exceptionfactory commented on code in PR #6392: URL: https://github.com/apache/nifi/pull/6392#discussion_r967660177
########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); + + public static final PropertyDescriptor SECRET_NAME_REGEX = new PropertyDescriptor.Builder() + .name("secret-name-regex") + .displayName("Secret Name Regex") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this Regex will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider service") + .description("The Controller Service that is used to obtain aws credentials provider") + .required(false) + .identifiesControllerService(AWSCredentialsProviderService.class) + .build(); + + public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder() + .name("aws-region") + .displayName("Region") + .required(true) + .allowableValues(getAvailableRegions()) + .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue()) + .build(); + + public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("aws-communications-timeout") + .displayName("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("aws-ssl-context-service") + .displayName("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static final String DEFAULT_USER_AGENT = "NiFi"; + private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + SECRET_NAME_REGEX, + REGION, + AWS_CREDENTIALS_PROVIDER_SERVICE, + TIMEOUT, + SSL_CONTEXT_SERVICE + )); + } + + @Override + public List<ParameterGroup> fetchParameters(final ConfigurationContext context) { + AWSSecretsManager secretsManager = this.configureClient(context); + + final List<ParameterGroup> groups = new ArrayList<>(); + final ListSecretsRequest listSecretsRequest = new ListSecretsRequest(); + final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest); + for (final SecretListEntry entry : listSecretsResult.getSecretList()) { + groups.addAll(fetchSecret(secretsManager, context, entry.getName())); + } + + return groups; + } + + @Override + public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) { + final List<ConfigVerificationResult> results = new ArrayList<>(); + + try { + final List<ParameterGroup> parameterGroups = fetchParameters(context); + int parameterCount = 0; + for (final ParameterGroup group : parameterGroups) { + parameterCount += group.getParameters().size(); + } + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Fetch Parameters") + .explanation(String.format("Fetched %s secret keys as parameters, across %s groups", + parameterCount, parameterGroups.size())) + .build()); + } catch (final Exception e) { + verificationLogger.error("Failed to fetch parameters", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Fetch Parameters") + .explanation("Failed to fetch parameters: " + e.getMessage()) + .build()); + } + return results; + } + + private List<ParameterGroup> fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) { + final List<ParameterGroup> groups = new ArrayList<>(); + final Pattern secretNamePattern = context.getProperty(SECRET_NAME_REGEX).isSet() + ? Pattern.compile(context.getProperty(SECRET_NAME_REGEX).getValue()) : null; + + final List<Parameter> parameters = new ArrayList<>(); + + if (!secretNamePattern.matcher(secretName).matches()) { + logger.debug("Secret [{}] does not match the secret name regex {}", secretName, secretNamePattern); + return groups; + } + + final GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest().withSecretId(secretName); + try { + final GetSecretValueResult getSecretValueResult = secretsManager.getSecretValue(getSecretValueRequest); + + if (getSecretValueResult.getSecretString() == null) { + logger.debug("Secret [{}] is not configured", secretName); + return groups; + } + + final ObjectNode secretObject = parseSecret(getSecretValueResult.getSecretString()); + if (secretObject == null) { + logger.debug("Secret [{}] is not in the expected JSON key/value format", secretName); + return groups; + } + + for (final Iterator<Map.Entry<String, JsonNode>> it = secretObject.fields(); it.hasNext(); ) { + final Map.Entry<String, JsonNode> field = it.next(); + final String parameterName = field.getKey(); + final String parameterValue = field.getValue().textValue(); + if (parameterValue == null) { + logger.debug("Secret [{}] key [{}] has no value", secretName, parameterValue); + continue; + } + + parameters.add(createParameter(parameterName, parameterValue)); + } + + groups.add(new ParameterGroup(secretName, parameters)); + + return groups; + } catch (final ResourceNotFoundException e) { + logger.error("Secret [{}] not found", secretName); + throw new IllegalStateException(String.format("Secret %s not found", secretName), e); + } catch (final AWSSecretsManagerException e) { + logger.error("Error retrieving secret [{}]", secretName); + throw new IllegalStateException("Error retrieving secret " + secretName, e); + } + } + + private Parameter createParameter(final String parameterName, final String parameterValue) { + final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build(); + return new Parameter(parameterDescriptor, parameterValue, null, true); + } + + protected ClientConfiguration createConfiguration(final ConfigurationContext context) { + final ClientConfiguration config = new ClientConfiguration(); + config.setMaxErrorRetry(0); + config.setUserAgentPrefix(DEFAULT_USER_AGENT); + config.setProtocol(DEFAULT_PROTOCOL); + final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); + config.setConnectionTimeout(commsTimeout); + config.setSocketTimeout(commsTimeout); + + if (this.getSupportedPropertyDescriptors().contains(SSL_CONTEXT_SERVICE)) { Review Comment: Is there a reason for this conditional? It should always evaluate to `true` based on the current implementation of `getSupportedPropertyDescriptors`. ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); + + public static final PropertyDescriptor SECRET_NAME_REGEX = new PropertyDescriptor.Builder() + .name("secret-name-regex") + .displayName("Secret Name Regex") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this Regex will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider service") + .description("The Controller Service that is used to obtain aws credentials provider") + .required(false) + .identifiesControllerService(AWSCredentialsProviderService.class) + .build(); + + public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder() + .name("aws-region") + .displayName("Region") + .required(true) + .allowableValues(getAvailableRegions()) + .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue()) + .build(); + + public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("aws-communications-timeout") + .displayName("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("aws-ssl-context-service") + .displayName("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static final String DEFAULT_USER_AGENT = "NiFi"; + private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + SECRET_NAME_REGEX, + REGION, + AWS_CREDENTIALS_PROVIDER_SERVICE, + TIMEOUT, + SSL_CONTEXT_SERVICE + )); + } + + @Override + public List<ParameterGroup> fetchParameters(final ConfigurationContext context) { + AWSSecretsManager secretsManager = this.configureClient(context); + + final List<ParameterGroup> groups = new ArrayList<>(); + final ListSecretsRequest listSecretsRequest = new ListSecretsRequest(); + final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest); + for (final SecretListEntry entry : listSecretsResult.getSecretList()) { + groups.addAll(fetchSecret(secretsManager, context, entry.getName())); + } + + return groups; + } + + @Override + public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) { + final List<ConfigVerificationResult> results = new ArrayList<>(); + + try { + final List<ParameterGroup> parameterGroups = fetchParameters(context); + int parameterCount = 0; + for (final ParameterGroup group : parameterGroups) { + parameterCount += group.getParameters().size(); + } + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Fetch Parameters") + .explanation(String.format("Fetched %s secret keys as parameters, across %s groups", + parameterCount, parameterGroups.size())) + .build()); + } catch (final Exception e) { + verificationLogger.error("Failed to fetch parameters", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Fetch Parameters") + .explanation("Failed to fetch parameters: " + e.getMessage()) + .build()); + } + return results; + } + + private List<ParameterGroup> fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) { + final List<ParameterGroup> groups = new ArrayList<>(); + final Pattern secretNamePattern = context.getProperty(SECRET_NAME_REGEX).isSet() + ? Pattern.compile(context.getProperty(SECRET_NAME_REGEX).getValue()) : null; + + final List<Parameter> parameters = new ArrayList<>(); + + if (!secretNamePattern.matcher(secretName).matches()) { + logger.debug("Secret [{}] does not match the secret name regex {}", secretName, secretNamePattern); + return groups; + } + + final GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest().withSecretId(secretName); + try { + final GetSecretValueResult getSecretValueResult = secretsManager.getSecretValue(getSecretValueRequest); + + if (getSecretValueResult.getSecretString() == null) { + logger.debug("Secret [{}] is not configured", secretName); + return groups; + } + + final ObjectNode secretObject = parseSecret(getSecretValueResult.getSecretString()); + if (secretObject == null) { + logger.debug("Secret [{}] is not in the expected JSON key/value format", secretName); + return groups; + } + + for (final Iterator<Map.Entry<String, JsonNode>> it = secretObject.fields(); it.hasNext(); ) { + final Map.Entry<String, JsonNode> field = it.next(); + final String parameterName = field.getKey(); + final String parameterValue = field.getValue().textValue(); + if (parameterValue == null) { + logger.debug("Secret [{}] key [{}] has no value", secretName, parameterValue); Review Comment: The second argument should be `parameterName`. ```suggestion logger.debug("Secret [{}] Parameter [{}] is not defined", secretName, parameterName); ``` ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.MockComponentLog; +import org.apache.nifi.util.MockConfigurationContext; +import org.apache.nifi.util.MockParameterProviderInitializationContext; +import org.junit.Test; +import org.mockito.ArgumentMatcher; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class TestAwsSecretsManagerParameterProvider { + final ObjectMapper objectMapper = new ObjectMapper(); + + final List<Parameter> mySecretParameters = Arrays.asList( + parameter("paramA", "valueA"), + parameter("paramB", "valueB"), + parameter("otherC", "valueOther"), + parameter("paramD", "valueD"), + parameter("nonSensitiveE", "valueE"), + parameter("otherF", "valueF") + ); + final List<Parameter> otherSecretParameters = Arrays.asList( + parameter("paramG", "valueG"), + parameter("otherH", "valueOther") + ); + final List<ParameterGroup> mockParameterGroups = Arrays.asList( + new ParameterGroup("MySecret", mySecretParameters), + new ParameterGroup("OtherSecret", otherSecretParameters) + ); + + private AwsSecretsManagerParameterProvider getParameterProvider() { + return spy(new AwsSecretsManagerParameterProvider()); + } + + private AWSSecretsManager mockSecretsManager(final List<ParameterGroup> mockGroup) { + final AWSSecretsManager secretsManager = mock(AWSSecretsManager.class); + + final ListSecretsResult listSecretsResult = mock(ListSecretsResult.class); + final List<SecretListEntry> secretList = mockGroup.stream() + .map(group -> new SecretListEntry().withName(group.getGroupName())) + .collect(Collectors.toList()); + when(listSecretsResult.getSecretList()).thenReturn(secretList); + when(secretsManager.listSecrets(any())).thenReturn(listSecretsResult); + + mockGroup.forEach(group -> { + final String groupName = group.getGroupName(); + final Map<String, String> keyValues = group.getParameters().stream().collect(Collectors.toMap( + param -> param.getDescriptor().getName(), + Parameter::getValue)); + final String secretString; + try { + secretString = objectMapper.writeValueAsString(keyValues); + final GetSecretValueResult result = new GetSecretValueResult().withName(groupName).withSecretString(secretString); + when(secretsManager.getSecretValue(argThat(matchesGetSecretValueRequest(groupName)))) + .thenReturn(result); + } catch (final JsonProcessingException e) { + // Ignore + } Review Comment: Instead of hiding the exception, it could be declared on the method and passed through to the test method ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.MockComponentLog; +import org.apache.nifi.util.MockConfigurationContext; +import org.apache.nifi.util.MockParameterProviderInitializationContext; +import org.junit.Test; +import org.mockito.ArgumentMatcher; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class TestAwsSecretsManagerParameterProvider { + final ObjectMapper objectMapper = new ObjectMapper(); + + final List<Parameter> mySecretParameters = Arrays.asList( + parameter("paramA", "valueA"), + parameter("paramB", "valueB"), + parameter("otherC", "valueOther"), + parameter("paramD", "valueD"), + parameter("nonSensitiveE", "valueE"), + parameter("otherF", "valueF") + ); + final List<Parameter> otherSecretParameters = Arrays.asList( + parameter("paramG", "valueG"), + parameter("otherH", "valueOther") + ); + final List<ParameterGroup> mockParameterGroups = Arrays.asList( + new ParameterGroup("MySecret", mySecretParameters), + new ParameterGroup("OtherSecret", otherSecretParameters) + ); + + private AwsSecretsManagerParameterProvider getParameterProvider() { + return spy(new AwsSecretsManagerParameterProvider()); + } + + private AWSSecretsManager mockSecretsManager(final List<ParameterGroup> mockGroup) { + final AWSSecretsManager secretsManager = mock(AWSSecretsManager.class); + + final ListSecretsResult listSecretsResult = mock(ListSecretsResult.class); + final List<SecretListEntry> secretList = mockGroup.stream() + .map(group -> new SecretListEntry().withName(group.getGroupName())) + .collect(Collectors.toList()); + when(listSecretsResult.getSecretList()).thenReturn(secretList); + when(secretsManager.listSecrets(any())).thenReturn(listSecretsResult); + + mockGroup.forEach(group -> { + final String groupName = group.getGroupName(); + final Map<String, String> keyValues = group.getParameters().stream().collect(Collectors.toMap( + param -> param.getDescriptor().getName(), + Parameter::getValue)); + final String secretString; + try { + secretString = objectMapper.writeValueAsString(keyValues); + final GetSecretValueResult result = new GetSecretValueResult().withName(groupName).withSecretString(secretString); + when(secretsManager.getSecretValue(argThat(matchesGetSecretValueRequest(groupName)))) + .thenReturn(result); + } catch (final JsonProcessingException e) { + // Ignore + } + }); + return secretsManager; + } + + private List<ParameterGroup> runProviderTest(final AWSSecretsManager secretsManager, final int expectedCount, + final ConfigVerificationResult.Outcome expectedOutcome) throws InitializationException { + + final AwsSecretsManagerParameterProvider parameterProvider = getParameterProvider(); + doReturn(secretsManager).when(parameterProvider).configureClient(any()); + final MockParameterProviderInitializationContext initContext = new MockParameterProviderInitializationContext("id", "name", + new MockComponentLog("providerId", parameterProvider)); + parameterProvider.initialize(initContext); + + final Map<PropertyDescriptor, String> properties = new HashMap<>(); + final MockConfigurationContext mockConfigurationContext = new MockConfigurationContext(properties, null); + + List<ParameterGroup> parameterGroups = new ArrayList<>(); + // Verify parameter fetching + if (expectedOutcome == ConfigVerificationResult.Outcome.FAILED) { + assertThrows(RuntimeException.class, () -> parameterProvider.fetchParameters(mockConfigurationContext)); + } else { + parameterGroups = parameterProvider.fetchParameters(mockConfigurationContext); + final int parameterCount = (int) parameterGroups.stream() + .flatMap(group -> group.getParameters().stream()) + .count(); + assertEquals(expectedCount, parameterCount); + } + + // Verify config verification + final List<ConfigVerificationResult> results = ((VerifiableParameterProvider) parameterProvider).verify(mockConfigurationContext, initContext.getLogger()); + + assertEquals(1, results.size()); + assertEquals(expectedOutcome, results.get(0).getOutcome()); + + return parameterGroups; + } + + @Test + public void testFetchParametersWithNoSecrets() throws InitializationException { + final List<ParameterGroup> expectedGroups = Collections.singletonList(new ParameterGroup("MySecret", Collections.emptyList())); + runProviderTest(mockSecretsManager(expectedGroups), 0, ConfigVerificationResult.Outcome.SUCCESSFUL); + } + + @Test + public void testFetchParameters() throws InitializationException { + runProviderTest(mockSecretsManager(mockParameterGroups), 8, ConfigVerificationResult.Outcome.SUCCESSFUL); + } + + @Test + public void testFetchParametersListFailure() throws InitializationException { + final AWSSecretsManager mockSecretsManager = mock(AWSSecretsManager.class); + when(mockSecretsManager.listSecrets(any())).thenThrow(new AWSSecretsManagerException("Fake exception")); + runProviderTest(mockSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED); + } + + @Test + public void testFetchParametersGetSecretFailure() throws InitializationException { + final AWSSecretsManager mockSecretsManager = mock(AWSSecretsManager.class); + final ListSecretsResult listSecretsResult = mock(ListSecretsResult.class); Review Comment: The `mock()` instantiation could be replaced with `@Mock` annotated member variables, together with `@ExtendWith(MockitoExtension.class)` on the class. ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); + + public static final PropertyDescriptor SECRET_NAME_REGEX = new PropertyDescriptor.Builder() + .name("secret-name-regex") + .displayName("Secret Name Regex") Review Comment: Recommend naming this `Secret Name Pattern`: ```suggestion public static final PropertyDescriptor SECRET_NAME_PATTERN = new PropertyDescriptor.Builder() .name("secret-name-pattern") .displayName("Secret Name Pattern") ``` ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); + + public static final PropertyDescriptor SECRET_NAME_REGEX = new PropertyDescriptor.Builder() + .name("secret-name-regex") + .displayName("Secret Name Regex") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this Regex will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider service") Review Comment: ```suggestion .displayName("AWS Credentials Provider Service") ``` ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); + + public static final PropertyDescriptor SECRET_NAME_REGEX = new PropertyDescriptor.Builder() + .name("secret-name-regex") + .displayName("Secret Name Regex") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this Regex will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider service") + .description("The Controller Service that is used to obtain aws credentials provider") + .required(false) + .identifiesControllerService(AWSCredentialsProviderService.class) + .build(); + + public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder() + .name("aws-region") + .displayName("Region") + .required(true) + .allowableValues(getAvailableRegions()) + .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue()) + .build(); + + public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("aws-communications-timeout") + .displayName("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("aws-ssl-context-service") + .displayName("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static final String DEFAULT_USER_AGENT = "NiFi"; + private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + SECRET_NAME_REGEX, + REGION, + AWS_CREDENTIALS_PROVIDER_SERVICE, + TIMEOUT, + SSL_CONTEXT_SERVICE + )); + } + + @Override + public List<ParameterGroup> fetchParameters(final ConfigurationContext context) { + AWSSecretsManager secretsManager = this.configureClient(context); + + final List<ParameterGroup> groups = new ArrayList<>(); + final ListSecretsRequest listSecretsRequest = new ListSecretsRequest(); + final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest); + for (final SecretListEntry entry : listSecretsResult.getSecretList()) { + groups.addAll(fetchSecret(secretsManager, context, entry.getName())); + } + + return groups; + } + + @Override + public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) { + final List<ConfigVerificationResult> results = new ArrayList<>(); + + try { + final List<ParameterGroup> parameterGroups = fetchParameters(context); + int parameterCount = 0; + for (final ParameterGroup group : parameterGroups) { + parameterCount += group.getParameters().size(); + } + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Fetch Parameters") + .explanation(String.format("Fetched %s secret keys as parameters, across %s groups", + parameterCount, parameterGroups.size())) + .build()); + } catch (final Exception e) { + verificationLogger.error("Failed to fetch parameters", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Fetch Parameters") + .explanation("Failed to fetch parameters: " + e.getMessage()) + .build()); + } + return results; + } + + private List<ParameterGroup> fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) { + final List<ParameterGroup> groups = new ArrayList<>(); + final Pattern secretNamePattern = context.getProperty(SECRET_NAME_REGEX).isSet() + ? Pattern.compile(context.getProperty(SECRET_NAME_REGEX).getValue()) : null; + + final List<Parameter> parameters = new ArrayList<>(); + + if (!secretNamePattern.matcher(secretName).matches()) { + logger.debug("Secret [{}] does not match the secret name regex {}", secretName, secretNamePattern); + return groups; + } + + final GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest().withSecretId(secretName); + try { + final GetSecretValueResult getSecretValueResult = secretsManager.getSecretValue(getSecretValueRequest); + + if (getSecretValueResult.getSecretString() == null) { + logger.debug("Secret [{}] is not configured", secretName); + return groups; + } + + final ObjectNode secretObject = parseSecret(getSecretValueResult.getSecretString()); + if (secretObject == null) { + logger.debug("Secret [{}] is not in the expected JSON key/value format", secretName); + return groups; + } + + for (final Iterator<Map.Entry<String, JsonNode>> it = secretObject.fields(); it.hasNext(); ) { + final Map.Entry<String, JsonNode> field = it.next(); + final String parameterName = field.getKey(); + final String parameterValue = field.getValue().textValue(); + if (parameterValue == null) { + logger.debug("Secret [{}] key [{}] has no value", secretName, parameterValue); + continue; + } + + parameters.add(createParameter(parameterName, parameterValue)); + } + + groups.add(new ParameterGroup(secretName, parameters)); + + return groups; + } catch (final ResourceNotFoundException e) { + logger.error("Secret [{}] not found", secretName); + throw new IllegalStateException(String.format("Secret %s not found", secretName), e); + } catch (final AWSSecretsManagerException e) { + logger.error("Error retrieving secret [{}]", secretName); + throw new IllegalStateException("Error retrieving secret " + secretName, e); + } + } + + private Parameter createParameter(final String parameterName, final String parameterValue) { + final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build(); + return new Parameter(parameterDescriptor, parameterValue, null, true); + } + + protected ClientConfiguration createConfiguration(final ConfigurationContext context) { + final ClientConfiguration config = new ClientConfiguration(); + config.setMaxErrorRetry(0); + config.setUserAgentPrefix(DEFAULT_USER_AGENT); + config.setProtocol(DEFAULT_PROTOCOL); + final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); + config.setConnectionTimeout(commsTimeout); + config.setSocketTimeout(commsTimeout); + + if (this.getSupportedPropertyDescriptors().contains(SSL_CONTEXT_SERVICE)) { + final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); + if (sslContextService != null) { + final SSLContext sslContext = sslContextService.createContext(); + SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, new DefaultHostnameVerifier()); + config.getApacheHttpClientConfig().setSslSocketFactory(sdkTLSSocketFactory); + } + } + + return config; + } + + public static AllowableValue createAllowableValue(final Regions region) { + return new AllowableValue(region.getName(), region.getDescription(), "AWS Region Code : " + region.getName()); + } + + public static AllowableValue[] getAvailableRegions() { + final List<AllowableValue> values = new ArrayList<>(); + for (final Regions region : Regions.values()) { + values.add(createAllowableValue(region)); + } + return values.toArray(new AllowableValue[0]); + } Review Comment: Recommend moving this methods higher in the file following general conventions for public static methods being declared before less visible methods. Do they need to `public` or should they be package-private or private? ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); + + public static final PropertyDescriptor SECRET_NAME_REGEX = new PropertyDescriptor.Builder() + .name("secret-name-regex") + .displayName("Secret Name Regex") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this Regex will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider service") + .description("The Controller Service that is used to obtain aws credentials provider") + .required(false) + .identifiesControllerService(AWSCredentialsProviderService.class) + .build(); + + public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder() + .name("aws-region") + .displayName("Region") + .required(true) + .allowableValues(getAvailableRegions()) + .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue()) + .build(); + + public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("aws-communications-timeout") + .displayName("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("aws-ssl-context-service") + .displayName("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static final String DEFAULT_USER_AGENT = "NiFi"; + private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + SECRET_NAME_REGEX, + REGION, + AWS_CREDENTIALS_PROVIDER_SERVICE, + TIMEOUT, + SSL_CONTEXT_SERVICE + )); + } + + @Override + public List<ParameterGroup> fetchParameters(final ConfigurationContext context) { + AWSSecretsManager secretsManager = this.configureClient(context); + + final List<ParameterGroup> groups = new ArrayList<>(); + final ListSecretsRequest listSecretsRequest = new ListSecretsRequest(); + final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest); + for (final SecretListEntry entry : listSecretsResult.getSecretList()) { + groups.addAll(fetchSecret(secretsManager, context, entry.getName())); + } + + return groups; + } + + @Override + public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) { + final List<ConfigVerificationResult> results = new ArrayList<>(); + + try { + final List<ParameterGroup> parameterGroups = fetchParameters(context); + int parameterCount = 0; + for (final ParameterGroup group : parameterGroups) { + parameterCount += group.getParameters().size(); + } + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Fetch Parameters") + .explanation(String.format("Fetched %s secret keys as parameters, across %s groups", + parameterCount, parameterGroups.size())) + .build()); + } catch (final Exception e) { + verificationLogger.error("Failed to fetch parameters", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Fetch Parameters") + .explanation("Failed to fetch parameters: " + e.getMessage()) + .build()); + } + return results; + } + + private List<ParameterGroup> fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) { + final List<ParameterGroup> groups = new ArrayList<>(); + final Pattern secretNamePattern = context.getProperty(SECRET_NAME_REGEX).isSet() + ? Pattern.compile(context.getProperty(SECRET_NAME_REGEX).getValue()) : null; + + final List<Parameter> parameters = new ArrayList<>(); + + if (!secretNamePattern.matcher(secretName).matches()) { + logger.debug("Secret [{}] does not match the secret name regex {}", secretName, secretNamePattern); + return groups; + } + + final GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest().withSecretId(secretName); + try { + final GetSecretValueResult getSecretValueResult = secretsManager.getSecretValue(getSecretValueRequest); + + if (getSecretValueResult.getSecretString() == null) { + logger.debug("Secret [{}] is not configured", secretName); + return groups; + } + + final ObjectNode secretObject = parseSecret(getSecretValueResult.getSecretString()); + if (secretObject == null) { + logger.debug("Secret [{}] is not in the expected JSON key/value format", secretName); + return groups; + } + + for (final Iterator<Map.Entry<String, JsonNode>> it = secretObject.fields(); it.hasNext(); ) { + final Map.Entry<String, JsonNode> field = it.next(); + final String parameterName = field.getKey(); + final String parameterValue = field.getValue().textValue(); + if (parameterValue == null) { + logger.debug("Secret [{}] key [{}] has no value", secretName, parameterValue); + continue; + } + + parameters.add(createParameter(parameterName, parameterValue)); + } + + groups.add(new ParameterGroup(secretName, parameters)); + + return groups; + } catch (final ResourceNotFoundException e) { + logger.error("Secret [{}] not found", secretName); + throw new IllegalStateException(String.format("Secret %s not found", secretName), e); + } catch (final AWSSecretsManagerException e) { + logger.error("Error retrieving secret [{}]", secretName); + throw new IllegalStateException("Error retrieving secret " + secretName, e); + } Review Comment: Recommend moving the `logger.error()` calls since the same information should be provided through the exceptions thrown. ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); + + public static final PropertyDescriptor SECRET_NAME_REGEX = new PropertyDescriptor.Builder() + .name("secret-name-regex") + .displayName("Secret Name Regex") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this Regex will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider service") + .description("The Controller Service that is used to obtain aws credentials provider") + .required(false) + .identifiesControllerService(AWSCredentialsProviderService.class) + .build(); + + public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder() + .name("aws-region") + .displayName("Region") + .required(true) + .allowableValues(getAvailableRegions()) + .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue()) + .build(); + + public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("aws-communications-timeout") + .displayName("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("aws-ssl-context-service") + .displayName("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static final String DEFAULT_USER_AGENT = "NiFi"; + private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + SECRET_NAME_REGEX, + REGION, + AWS_CREDENTIALS_PROVIDER_SERVICE, + TIMEOUT, + SSL_CONTEXT_SERVICE + )); + } + + @Override + public List<ParameterGroup> fetchParameters(final ConfigurationContext context) { + AWSSecretsManager secretsManager = this.configureClient(context); + + final List<ParameterGroup> groups = new ArrayList<>(); + final ListSecretsRequest listSecretsRequest = new ListSecretsRequest(); + final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest); + for (final SecretListEntry entry : listSecretsResult.getSecretList()) { + groups.addAll(fetchSecret(secretsManager, context, entry.getName())); + } + + return groups; + } + + @Override + public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) { + final List<ConfigVerificationResult> results = new ArrayList<>(); + + try { + final List<ParameterGroup> parameterGroups = fetchParameters(context); + int parameterCount = 0; + for (final ParameterGroup group : parameterGroups) { + parameterCount += group.getParameters().size(); + } + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Fetch Parameters") + .explanation(String.format("Fetched %s secret keys as parameters, across %s groups", + parameterCount, parameterGroups.size())) + .build()); + } catch (final Exception e) { + verificationLogger.error("Failed to fetch parameters", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Fetch Parameters") + .explanation("Failed to fetch parameters: " + e.getMessage()) + .build()); + } + return results; + } + + private List<ParameterGroup> fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) { + final List<ParameterGroup> groups = new ArrayList<>(); + final Pattern secretNamePattern = context.getProperty(SECRET_NAME_REGEX).isSet() + ? Pattern.compile(context.getProperty(SECRET_NAME_REGEX).getValue()) : null; + + final List<Parameter> parameters = new ArrayList<>(); + + if (!secretNamePattern.matcher(secretName).matches()) { + logger.debug("Secret [{}] does not match the secret name regex {}", secretName, secretNamePattern); Review Comment: ```suggestion logger.debug("Secret [{}] does not match the secret name pattern {}", secretName, secretNamePattern); ``` ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); + + public static final PropertyDescriptor SECRET_NAME_REGEX = new PropertyDescriptor.Builder() + .name("secret-name-regex") + .displayName("Secret Name Regex") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this Regex will not be fetched.") Review Comment: ```suggestion "Any secrets whose names do not match this pattern will not be fetched.") ``` ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); + + public static final PropertyDescriptor SECRET_NAME_REGEX = new PropertyDescriptor.Builder() + .name("secret-name-regex") + .displayName("Secret Name Regex") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this Regex will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider service") + .description("The Controller Service that is used to obtain aws credentials provider") Review Comment: ```suggestion .description("Service used to obtain an Amazon Web Services Credentials Provider") ``` ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); Review Comment: Is the ComponentLog available? It should be used in place of this logger. ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + private static final Logger logger = LoggerFactory.getLogger(AwsSecretsManagerParameterProvider.class); + + public static final PropertyDescriptor SECRET_NAME_REGEX = new PropertyDescriptor.Builder() + .name("secret-name-regex") + .displayName("Secret Name Regex") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this Regex will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider service") + .description("The Controller Service that is used to obtain aws credentials provider") + .required(false) + .identifiesControllerService(AWSCredentialsProviderService.class) + .build(); + + public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder() + .name("aws-region") + .displayName("Region") + .required(true) + .allowableValues(getAvailableRegions()) + .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue()) + .build(); + + public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("aws-communications-timeout") + .displayName("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("aws-ssl-context-service") + .displayName("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static final String DEFAULT_USER_AGENT = "NiFi"; + private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + SECRET_NAME_REGEX, + REGION, + AWS_CREDENTIALS_PROVIDER_SERVICE, + TIMEOUT, + SSL_CONTEXT_SERVICE + )); + } + + @Override + public List<ParameterGroup> fetchParameters(final ConfigurationContext context) { + AWSSecretsManager secretsManager = this.configureClient(context); + + final List<ParameterGroup> groups = new ArrayList<>(); + final ListSecretsRequest listSecretsRequest = new ListSecretsRequest(); + final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest); + for (final SecretListEntry entry : listSecretsResult.getSecretList()) { + groups.addAll(fetchSecret(secretsManager, context, entry.getName())); + } + + return groups; + } + + @Override + public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) { + final List<ConfigVerificationResult> results = new ArrayList<>(); + + try { + final List<ParameterGroup> parameterGroups = fetchParameters(context); + int parameterCount = 0; + for (final ParameterGroup group : parameterGroups) { + parameterCount += group.getParameters().size(); + } + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Fetch Parameters") + .explanation(String.format("Fetched %s secret keys as parameters, across %s groups", + parameterCount, parameterGroups.size())) + .build()); + } catch (final Exception e) { + verificationLogger.error("Failed to fetch parameters", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Fetch Parameters") + .explanation("Failed to fetch parameters: " + e.getMessage()) + .build()); + } + return results; + } + + private List<ParameterGroup> fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) { + final List<ParameterGroup> groups = new ArrayList<>(); + final Pattern secretNamePattern = context.getProperty(SECRET_NAME_REGEX).isSet() + ? Pattern.compile(context.getProperty(SECRET_NAME_REGEX).getValue()) : null; + + final List<Parameter> parameters = new ArrayList<>(); + + if (!secretNamePattern.matcher(secretName).matches()) { + logger.debug("Secret [{}] does not match the secret name regex {}", secretName, secretNamePattern); + return groups; + } + + final GetSecretValueRequest getSecretValueRequest = new GetSecretValueRequest().withSecretId(secretName); + try { + final GetSecretValueResult getSecretValueResult = secretsManager.getSecretValue(getSecretValueRequest); + + if (getSecretValueResult.getSecretString() == null) { + logger.debug("Secret [{}] is not configured", secretName); + return groups; + } + + final ObjectNode secretObject = parseSecret(getSecretValueResult.getSecretString()); + if (secretObject == null) { + logger.debug("Secret [{}] is not in the expected JSON key/value format", secretName); + return groups; + } + + for (final Iterator<Map.Entry<String, JsonNode>> it = secretObject.fields(); it.hasNext(); ) { + final Map.Entry<String, JsonNode> field = it.next(); + final String parameterName = field.getKey(); + final String parameterValue = field.getValue().textValue(); + if (parameterValue == null) { + logger.debug("Secret [{}] key [{}] has no value", secretName, parameterValue); + continue; + } + + parameters.add(createParameter(parameterName, parameterValue)); + } + + groups.add(new ParameterGroup(secretName, parameters)); + + return groups; + } catch (final ResourceNotFoundException e) { + logger.error("Secret [{}] not found", secretName); + throw new IllegalStateException(String.format("Secret %s not found", secretName), e); + } catch (final AWSSecretsManagerException e) { + logger.error("Error retrieving secret [{}]", secretName); + throw new IllegalStateException("Error retrieving secret " + secretName, e); + } + } + + private Parameter createParameter(final String parameterName, final String parameterValue) { + final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build(); + return new Parameter(parameterDescriptor, parameterValue, null, true); + } + + protected ClientConfiguration createConfiguration(final ConfigurationContext context) { + final ClientConfiguration config = new ClientConfiguration(); + config.setMaxErrorRetry(0); + config.setUserAgentPrefix(DEFAULT_USER_AGENT); + config.setProtocol(DEFAULT_PROTOCOL); + final int commsTimeout = context.getProperty(TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue(); + config.setConnectionTimeout(commsTimeout); + config.setSocketTimeout(commsTimeout); + + if (this.getSupportedPropertyDescriptors().contains(SSL_CONTEXT_SERVICE)) { + final SSLContextService sslContextService = context.getProperty(SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class); + if (sslContextService != null) { + final SSLContext sslContext = sslContextService.createContext(); + SdkTLSSocketFactory sdkTLSSocketFactory = new SdkTLSSocketFactory(sslContext, new DefaultHostnameVerifier()); Review Comment: Instead of instantiating a new `DefaultHostnameVerifier`, it looks one of the static properties on `SdkTLSSocketFactory` could be used. The Browser Compatible version seems like the best option. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
