exceptionfactory commented on code in PR #6392: URL: https://github.com/apache/nifi/pull/6392#discussion_r983022037
########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + + public static final PropertyDescriptor SECRET_NAME_PATTERN = new PropertyDescriptor.Builder() + .name("secret-name-pattern") + .displayName("Secret Name Pattern") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this pattern will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider Service") + .description("Service used to obtain an Amazon Web Services Credentials Provider") + .required(false) + .identifiesControllerService(AWSCredentialsProviderService.class) + .build(); + + public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder() + .name("aws-region") + .displayName("Region") + .required(true) + .allowableValues(getAvailableRegions()) + .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue()) + .build(); + + public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("aws-communications-timeout") + .displayName("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("aws-ssl-context-service") + .displayName("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static final String DEFAULT_USER_AGENT = "NiFi"; + private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + SECRET_NAME_PATTERN, + REGION, + AWS_CREDENTIALS_PROVIDER_SERVICE, + TIMEOUT, + SSL_CONTEXT_SERVICE + )); + } + + @Override + public List<ParameterGroup> fetchParameters(final ConfigurationContext context) { + AWSSecretsManager secretsManager = this.configureClient(context); + + final List<ParameterGroup> groups = new ArrayList<>(); + final ListSecretsRequest listSecretsRequest = new ListSecretsRequest(); + final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest); + for (final SecretListEntry entry : listSecretsResult.getSecretList()) { + groups.addAll(fetchSecret(secretsManager, context, entry.getName())); + } + + return groups; + } + + @Override + public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) { + final List<ConfigVerificationResult> results = new ArrayList<>(); + + try { + final List<ParameterGroup> parameterGroups = fetchParameters(context); + int parameterCount = 0; + for (final ParameterGroup group : parameterGroups) { + parameterCount += group.getParameters().size(); + } + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Fetch Parameters") + .explanation(String.format("Fetched %s secret keys as parameters, across %s groups", Review Comment: The `%s` placeholders should be replaced with `%d` since the values are integers. Recommend a slight wording adjustment to handle 1. ```suggestion .explanation(String.format("Fetched secret keys [%d] as parameters, across groups [%d]", ``` ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + + public static final PropertyDescriptor SECRET_NAME_PATTERN = new PropertyDescriptor.Builder() + .name("secret-name-pattern") + .displayName("Secret Name Pattern") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this pattern will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider Service") + .description("Service used to obtain an Amazon Web Services Credentials Provider") + .required(false) Review Comment: It looks like this property should be required. ```suggestion .required(true) ``` ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.MockComponentLog; +import org.apache.nifi.util.MockConfigurationContext; +import org.apache.nifi.util.MockParameterProviderInitializationContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatcher; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; Review Comment: These should be changed to JUnit Jupiter references. ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + + public static final PropertyDescriptor SECRET_NAME_PATTERN = new PropertyDescriptor.Builder() + .name("secret-name-pattern") + .displayName("Secret Name Pattern") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this pattern will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider Service") + .description("Service used to obtain an Amazon Web Services Credentials Provider") + .required(false) + .identifiesControllerService(AWSCredentialsProviderService.class) + .build(); + + public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder() + .name("aws-region") + .displayName("Region") + .required(true) + .allowableValues(getAvailableRegions()) + .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue()) + .build(); + + public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("aws-communications-timeout") + .displayName("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("aws-ssl-context-service") + .displayName("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static final String DEFAULT_USER_AGENT = "NiFi"; + private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + SECRET_NAME_PATTERN, + REGION, + AWS_CREDENTIALS_PROVIDER_SERVICE, + TIMEOUT, + SSL_CONTEXT_SERVICE + )); + } + + @Override + public List<ParameterGroup> fetchParameters(final ConfigurationContext context) { + AWSSecretsManager secretsManager = this.configureClient(context); + + final List<ParameterGroup> groups = new ArrayList<>(); + final ListSecretsRequest listSecretsRequest = new ListSecretsRequest(); + final ListSecretsResult listSecretsResult = secretsManager.listSecrets(listSecretsRequest); + for (final SecretListEntry entry : listSecretsResult.getSecretList()) { + groups.addAll(fetchSecret(secretsManager, context, entry.getName())); + } + + return groups; + } + + @Override + public List<ConfigVerificationResult> verify(final ConfigurationContext context, final ComponentLog verificationLogger) { + final List<ConfigVerificationResult> results = new ArrayList<>(); + + try { + final List<ParameterGroup> parameterGroups = fetchParameters(context); + int parameterCount = 0; + for (final ParameterGroup group : parameterGroups) { + parameterCount += group.getParameters().size(); + } + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .verificationStepName("Fetch Parameters") + .explanation(String.format("Fetched %s secret keys as parameters, across %s groups", + parameterCount, parameterGroups.size())) + .build()); + } catch (final Exception e) { + verificationLogger.error("Failed to fetch parameters", e); + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Fetch Parameters") + .explanation("Failed to fetch parameters: " + e.getMessage()) + .build()); + } + return results; + } + + private List<ParameterGroup> fetchSecret(final AWSSecretsManager secretsManager, final ConfigurationContext context, final String secretName) { + final List<ParameterGroup> groups = new ArrayList<>(); + final Pattern secretNamePattern = context.getProperty(SECRET_NAME_PATTERN).isSet() + ? Pattern.compile(context.getProperty(SECRET_NAME_PATTERN).getValue()) : null; Review Comment: It looks like this could set `secretNamePattern` to `null`, but `secretNamePattern.matcher()` expects it to be defined. In this case, the default value should be a static default `Pattern` instance. ```suggestion final Pattern secretNamePattern = context.getProperty(SECRET_NAME_PATTERN).isSet() ? Pattern.compile(context.getProperty(SECRET_NAME_PATTERN).getValue()) : DEFAULT_PATTERN; ``` ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.MockComponentLog; +import org.apache.nifi.util.MockConfigurationContext; +import org.apache.nifi.util.MockParameterProviderInitializationContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatcher; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class TestAwsSecretsManagerParameterProvider { + + @Mock + private AWSSecretsManager defaultSecretsManager; + + @Mock + private ListSecretsResult listSecretsResult; + + final ObjectMapper objectMapper = new ObjectMapper(); + + final List<Parameter> mySecretParameters = Arrays.asList( + parameter("paramA", "valueA"), + parameter("paramB", "valueB"), + parameter("otherC", "valueOther"), + parameter("paramD", "valueD"), + parameter("nonSensitiveE", "valueE"), + parameter("otherF", "valueF") + ); + final List<Parameter> otherSecretParameters = Arrays.asList( + parameter("paramG", "valueG"), + parameter("otherH", "valueOther") + ); + final List<ParameterGroup> mockParameterGroups = Arrays.asList( + new ParameterGroup("MySecret", mySecretParameters), + new ParameterGroup("OtherSecret", otherSecretParameters) + ); + + private AwsSecretsManagerParameterProvider getParameterProvider() { + return spy(new AwsSecretsManagerParameterProvider()); + } + + private AWSSecretsManager mockSecretsManager(final List<ParameterGroup> mockGroup) { + final AWSSecretsManager secretsManager = mock(AWSSecretsManager.class); + + final List<SecretListEntry> secretList = mockGroup.stream() + .map(group -> new SecretListEntry().withName(group.getGroupName())) + .collect(Collectors.toList()); + when(listSecretsResult.getSecretList()).thenReturn(secretList); + when(secretsManager.listSecrets(any())).thenReturn(listSecretsResult); + + mockGroup.forEach(group -> { + final String groupName = group.getGroupName(); + final Map<String, String> keyValues = group.getParameters().stream().collect(Collectors.toMap( + param -> param.getDescriptor().getName(), + Parameter::getValue)); + final String secretString; + try { + secretString = objectMapper.writeValueAsString(keyValues); + final GetSecretValueResult result = new GetSecretValueResult().withName(groupName).withSecretString(secretString); + when(secretsManager.getSecretValue(argThat(matchesGetSecretValueRequest(groupName)))) + .thenReturn(result); + } catch (final JsonProcessingException e) { + throw new IllegalStateException(e); + } + }); + return secretsManager; + } + + private List<ParameterGroup> runProviderTest(final AWSSecretsManager secretsManager, final int expectedCount, + final ConfigVerificationResult.Outcome expectedOutcome) throws InitializationException { + + final AwsSecretsManagerParameterProvider parameterProvider = getParameterProvider(); + doReturn(secretsManager).when(parameterProvider).configureClient(any()); + final MockParameterProviderInitializationContext initContext = new MockParameterProviderInitializationContext("id", "name", + new MockComponentLog("providerId", parameterProvider)); + parameterProvider.initialize(initContext); + + final Map<PropertyDescriptor, String> properties = new HashMap<>(); + final MockConfigurationContext mockConfigurationContext = new MockConfigurationContext(properties, null); + + List<ParameterGroup> parameterGroups = new ArrayList<>(); + // Verify parameter fetching + if (expectedOutcome == ConfigVerificationResult.Outcome.FAILED) { + assertThrows(RuntimeException.class, () -> parameterProvider.fetchParameters(mockConfigurationContext)); + } else { + parameterGroups = parameterProvider.fetchParameters(mockConfigurationContext); + final int parameterCount = (int) parameterGroups.stream() + .flatMap(group -> group.getParameters().stream()) + .count(); + assertEquals(expectedCount, parameterCount); + } + + // Verify config verification + final List<ConfigVerificationResult> results = ((VerifiableParameterProvider) parameterProvider).verify(mockConfigurationContext, initContext.getLogger()); + + assertEquals(1, results.size()); + assertEquals(expectedOutcome, results.get(0).getOutcome()); + + return parameterGroups; + } + + @Test + public void testFetchParametersWithNoSecrets() throws InitializationException { + final List<ParameterGroup> expectedGroups = Collections.singletonList(new ParameterGroup("MySecret", Collections.emptyList())); + runProviderTest(mockSecretsManager(expectedGroups), 0, ConfigVerificationResult.Outcome.SUCCESSFUL); + } + + @Test + public void testFetchParameters() throws InitializationException { + runProviderTest(mockSecretsManager(mockParameterGroups), 8, ConfigVerificationResult.Outcome.SUCCESSFUL); + } + + @Test + public void testFetchParametersListFailure() throws InitializationException { + when(defaultSecretsManager.listSecrets(any())).thenThrow(new AWSSecretsManagerException("Fake exception")); + runProviderTest(defaultSecretsManager, 0, ConfigVerificationResult.Outcome.FAILED); + } + + @Test + public void testFetchParametersGetSecretFailure() throws InitializationException { Review Comment: Recommend moving these Test-annotated methods before all other methods in the class. ########## nifi-nar-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java: ########## @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter.aws; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.secretsmanager.AWSSecretsManager; +import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder; +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException; +import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest; +import com.amazonaws.services.secretsmanager.model.GetSecretValueResult; +import com.amazonaws.services.secretsmanager.model.ListSecretsRequest; +import com.amazonaws.services.secretsmanager.model.ListSecretsResult; +import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException; +import com.amazonaws.services.secretsmanager.model.SecretListEntry; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.AllowableValue; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.parameter.AbstractParameterProvider; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterGroup; +import org.apache.nifi.parameter.VerifiableParameterProvider; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; +import org.apache.nifi.ssl.SSLContextService; + +import javax.net.ssl.SSLContext; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +/** + * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command: <br/><br/> + * <code>aws secretsmanager create-secret --name "[Context]" --secret-string '{ "[Param]": "[secretValue]", "[Param2]": "[secretValue2]" }'</code> <br/><br/> + * + */ + +@Tags({"aws", "secretsmanager", "secrets", "manager"}) +@CapabilityDescription("Fetches parameters from AWS SecretsManager. Each secret becomes a Parameter group, which can map to a Parameter Context, with " + + "key/value pairs in the secret mapping to Parameters in the group.") +public class AwsSecretsManagerParameterProvider extends AbstractParameterProvider implements VerifiableParameterProvider { + + public static final PropertyDescriptor SECRET_NAME_PATTERN = new PropertyDescriptor.Builder() + .name("secret-name-pattern") + .displayName("Secret Name Pattern") + .description("A Regular Expression matching on Secret Name that identifies Secrets whose parameters should be fetched. " + + "Any secrets whose names do not match this pattern will not be fetched.") + .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR) + .required(true) + .defaultValue(".*") + .build(); + /** + * AWS credentials provider service + * + * @see <a href="http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html";>AWSCredentialsProvider</a> + */ + public static final PropertyDescriptor AWS_CREDENTIALS_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("aws-credentials-provider-service") + .displayName("AWS Credentials Provider Service") + .description("Service used to obtain an Amazon Web Services Credentials Provider") + .required(false) + .identifiesControllerService(AWSCredentialsProviderService.class) + .build(); + + public static final PropertyDescriptor REGION = new PropertyDescriptor.Builder() + .name("aws-region") + .displayName("Region") + .required(true) + .allowableValues(getAvailableRegions()) + .defaultValue(createAllowableValue(Regions.DEFAULT_REGION).getValue()) + .build(); + + public static final PropertyDescriptor TIMEOUT = new PropertyDescriptor.Builder() + .name("aws-communications-timeout") + .displayName("Communications Timeout") + .required(true) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .defaultValue("30 secs") + .build(); + + public static final PropertyDescriptor SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("aws-ssl-context-service") + .displayName("SSL Context Service") + .description("Specifies an optional SSL Context Service that, if provided, will be used to create connections") + .required(false) + .identifiesControllerService(SSLContextService.class) + .build(); + + private static final String DEFAULT_USER_AGENT = "NiFi"; + private static final Protocol DEFAULT_PROTOCOL = Protocol.HTTPS; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Override + protected List<PropertyDescriptor> getSupportedPropertyDescriptors() { + return Collections.unmodifiableList(Arrays.asList( + SECRET_NAME_PATTERN, + REGION, + AWS_CREDENTIALS_PROVIDER_SERVICE, + TIMEOUT, + SSL_CONTEXT_SERVICE + )); Review Comment: It looks like this array could be declared in a static initializer. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
