jfrazee commented on a change in pull request #5078:
URL: https://github.com/apache/nifi/pull/5078#discussion_r635556255
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITListAzureBlobStorage.java
##########
@@ -20,11 +20,18 @@
import org.apache.nifi.util.MockFlowFile;
import org.junit.Before;
import org.junit.Test;
-
+// import org.junit.runner.RunWith;
+// import org.junit.runners.Parameterized;
import java.util.concurrent.TimeUnit;
+// @RunWith(Parameterized.class)
Review comment:
```suggestion
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
+ .required(true)
+ .allowableValues(buildCseEncryptionMethodAllowableValues())
+ .defaultValue(AzureBlobClientSideEncryptionMethod.NONE.name())
+ .description("Specifies the key type to use for client-side
encryption.")
+ .build();
+
+ public static final PropertyDescriptor CSE_KEY_ID = new
PropertyDescriptor.Builder()
+ .name("cse-key-id")
+ .displayName("Client-Side Encryption key ID")
+ .description("Specifies the ID of the key to use for client-side
encryption.")
+
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CSE_SYMMETRIC_KEY_HEX = new
PropertyDescriptor.Builder()
+ .name("cse-symmetric-key-hex")
+ .displayName("Symmetric Key (hexadecimal)")
+ .description("When using symmetric client-side encryption, this is
the raw key, encoded in hexadecimal")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(true)
+ .build();
+
+ private static AllowableValue[] buildCseEncryptionMethodAllowableValues() {
+ final AzureBlobClientSideEncryptionMethod[] encryptionMethods =
AzureBlobClientSideEncryptionMethod.values();
+ List<AllowableValue> allowableValues = new
ArrayList<>(encryptionMethods.length);
+ for (AzureBlobClientSideEncryptionMethod em : encryptionMethods) {
+ allowableValues.add(new AllowableValue(em.name(), em.name(),
em.toString()));
+ }
+
+ return allowableValues.toArray(new AllowableValue[0]);
+ }
+
+ public static Collection<ValidationResult>
validateClientSideEncryptionProperties(ValidationContext validationContext) {
+ final List<ValidationResult> validationResults = new ArrayList<>();
+
+ final String cseKeyTypeValue =
validationContext.getProperty(CSE_KEY_TYPE).getValue();
+ final AzureBlobClientSideEncryptionMethod cseKeyType =
AzureBlobClientSideEncryptionMethod.valueOf(cseKeyTypeValue);
+
+ final String cseKeyId =
validationContext.getProperty(CSE_KEY_ID).getValue();
+
+ final String cseSymmetricKeyHex =
validationContext.getProperty(CSE_SYMMETRIC_KEY_HEX).getValue();
+
+ if (cseKeyType != AzureBlobClientSideEncryptionMethod.NONE &&
Strings.isNullOrEmpty(cseKeyId)) {
+ validationResults.add(new
ValidationResult.Builder().subject(CSE_KEY_ID.getDisplayName())
+ .explanation("a key ID must be set when client-side
encryption is enabled.").build());
+ }
+
+ if (cseKeyType == AzureBlobClientSideEncryptionMethod.SYMMETRIC) {
+ validationResults.addAll(validateSymmetricKey(cseSymmetricKeyHex));
+ }
+
+ return validationResults;
+ }
+
+ private static List<ValidationResult> validateSymmetricKey(String keyHex) {
+ List<ValidationResult> validationResults = new ArrayList<>();
+ if (Strings.isNullOrEmpty(keyHex)) {
Review comment:
```suggestion
if (StringUtils.isBlank(keyHex)) {
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureBlobStorage.java
##########
@@ -48,6 +59,74 @@ public void testPutBlob() throws Exception {
assertResult();
}
+ @Test
+ public void testPutBlob64BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_64B_VALUE);
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testPutBlob128BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_128B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
+ runner.run();
+
+ assertResult();
+ }
+
+ @Test
+ public void testPutBlob192BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_192B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
+ runner.run();
+
+ assertResult();
+ }
+
+ @Test
+ public void testPutBlob256BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_256B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
+ runner.run();
+
+ assertResult();
+ }
+
+ @Test
+ public void testPutBlob384BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_384B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
+ runner.run();
+
+ assertResult();
+ }
+
+ @Test
+ public void testPutBlob512BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_512B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
Review comment:
```suggestion
runner.enqueue(TEST_FILE_CONTENT.getBytes());
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
Review comment:
Can we make `CSE_KEY_ID` and `CSE_SYMMETRIC_KEY_HEX` depend on
`CSE_KEY_TYPE` using dependent properties (see
[DependOnProperties.java](https://github.com/apache/nifi/blob/rel/nifi-1.13.2/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/processors/tests/system/DependOnProperties.java)
for an example)?
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITListAzureBlobStorage.java
##########
@@ -20,11 +20,18 @@
import org.apache.nifi.util.MockFlowFile;
import org.junit.Before;
import org.junit.Test;
-
+// import org.junit.runner.RunWith;
+// import org.junit.runners.Parameterized;
import java.util.concurrent.TimeUnit;
+// @RunWith(Parameterized.class)
public class ITListAzureBlobStorage extends AbstractAzureBlobStorageIT {
+ /* @Parameterized.Parameters
+ public static Object[][] data() {
+ return new Object[10][0];
+ }*/
+
Review comment:
```suggestion
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
+ .required(true)
+ .allowableValues(buildCseEncryptionMethodAllowableValues())
+ .defaultValue(AzureBlobClientSideEncryptionMethod.NONE.name())
+ .description("Specifies the key type to use for client-side
encryption.")
+ .build();
+
+ public static final PropertyDescriptor CSE_KEY_ID = new
PropertyDescriptor.Builder()
+ .name("cse-key-id")
+ .displayName("Client-Side Encryption key ID")
+ .description("Specifies the ID of the key to use for client-side
encryption.")
+
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
Review comment:
I think all whitespace should be illegal here too, right? If so
`NON_BLANK_VALIDATOR` is preferred. (Similar comment elsewhere if this is
right.)
```suggestion
.addValidator(StandardValidators.NON_BLANK_VALIDATOR)
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITListAzureBlobStorage.java
##########
@@ -20,11 +20,18 @@
import org.apache.nifi.util.MockFlowFile;
import org.junit.Before;
import org.junit.Test;
-
+// import org.junit.runner.RunWith;
+// import org.junit.runners.Parameterized;
Review comment:
```suggestion
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
+ .required(true)
+ .allowableValues(buildCseEncryptionMethodAllowableValues())
+ .defaultValue(AzureBlobClientSideEncryptionMethod.NONE.name())
+ .description("Specifies the key type to use for client-side
encryption.")
+ .build();
+
+ public static final PropertyDescriptor CSE_KEY_ID = new
PropertyDescriptor.Builder()
+ .name("cse-key-id")
+ .displayName("Client-Side Encryption key ID")
+ .description("Specifies the ID of the key to use for client-side
encryption.")
+
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CSE_SYMMETRIC_KEY_HEX = new
PropertyDescriptor.Builder()
+ .name("cse-symmetric-key-hex")
+ .displayName("Symmetric Key (hexadecimal)")
+ .description("When using symmetric client-side encryption, this is
the raw key, encoded in hexadecimal")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(true)
+ .build();
+
+ private static AllowableValue[] buildCseEncryptionMethodAllowableValues() {
+ final AzureBlobClientSideEncryptionMethod[] encryptionMethods =
AzureBlobClientSideEncryptionMethod.values();
+ List<AllowableValue> allowableValues = new
ArrayList<>(encryptionMethods.length);
+ for (AzureBlobClientSideEncryptionMethod em : encryptionMethods) {
+ allowableValues.add(new AllowableValue(em.name(), em.name(),
em.toString()));
+ }
+
+ return allowableValues.toArray(new AllowableValue[0]);
Review comment:
Can we do this as a stream? I think it'll be simpler:
```suggestion
return Arrays.stream(AzureBlobClientSideEncryptionMethod.values())
.map(v -> new AllowableValue(v.name(), v.getName(),
v.getDescription))
.toArray(AllowableValue[]::new);
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/FetchAzureBlobStorage.java
##########
@@ -112,11 +139,20 @@ public void onTrigger(ProcessContext context,
ProcessSession session) throws Pro
final Map<String, String> attributes = new HashMap<>();
final CloudBlob blob = container.getBlockBlobReference(blobPath);
+ BlobRequestOptions blobRequestOptions = new BlobRequestOptions();
+
+ if (cseKeyType == AzureBlobClientSideEncryptionMethod.SYMMETRIC) {
+ byte[] keyBytes =
Hex.decodeHex(cseSymmetricKeyHex.toCharArray());
+ SymmetricKey key = new SymmetricKey(cseKeyId, keyBytes);
+ BlobEncryptionPolicy policy = new BlobEncryptionPolicy(key,
null);
+ blobRequestOptions.setEncryptionPolicy(policy);
+ }
Review comment:
I might consider moving this into a `createBlobRequestOptions(context)`
or something.
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureBlobStorage.java
##########
@@ -48,6 +59,74 @@ public void testPutBlob() throws Exception {
assertResult();
}
+ @Test
+ public void testPutBlob64BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_64B_VALUE);
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testPutBlob128BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_128B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
Review comment:
```suggestion
runner.enqueue(TEST_FILE_CONTENT.getBytes());
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
+ .required(true)
+ .allowableValues(buildCseEncryptionMethodAllowableValues())
+ .defaultValue(AzureBlobClientSideEncryptionMethod.NONE.name())
+ .description("Specifies the key type to use for client-side
encryption.")
+ .build();
+
+ public static final PropertyDescriptor CSE_KEY_ID = new
PropertyDescriptor.Builder()
+ .name("cse-key-id")
+ .displayName("Client-Side Encryption key ID")
+ .description("Specifies the ID of the key to use for client-side
encryption.")
+
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CSE_SYMMETRIC_KEY_HEX = new
PropertyDescriptor.Builder()
+ .name("cse-symmetric-key-hex")
+ .displayName("Symmetric Key (hexadecimal)")
+ .description("When using symmetric client-side encryption, this is
the raw key, encoded in hexadecimal")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(true)
+ .build();
+
+ private static AllowableValue[] buildCseEncryptionMethodAllowableValues() {
+ final AzureBlobClientSideEncryptionMethod[] encryptionMethods =
AzureBlobClientSideEncryptionMethod.values();
+ List<AllowableValue> allowableValues = new
ArrayList<>(encryptionMethods.length);
+ for (AzureBlobClientSideEncryptionMethod em : encryptionMethods) {
+ allowableValues.add(new AllowableValue(em.name(), em.name(),
em.toString()));
+ }
+
+ return allowableValues.toArray(new AllowableValue[0]);
+ }
+
+ public static Collection<ValidationResult>
validateClientSideEncryptionProperties(ValidationContext validationContext) {
+ final List<ValidationResult> validationResults = new ArrayList<>();
+
+ final String cseKeyTypeValue =
validationContext.getProperty(CSE_KEY_TYPE).getValue();
+ final AzureBlobClientSideEncryptionMethod cseKeyType =
AzureBlobClientSideEncryptionMethod.valueOf(cseKeyTypeValue);
+
+ final String cseKeyId =
validationContext.getProperty(CSE_KEY_ID).getValue();
+
+ final String cseSymmetricKeyHex =
validationContext.getProperty(CSE_SYMMETRIC_KEY_HEX).getValue();
+
+ if (cseKeyType != AzureBlobClientSideEncryptionMethod.NONE &&
Strings.isNullOrEmpty(cseKeyId)) {
+ validationResults.add(new
ValidationResult.Builder().subject(CSE_KEY_ID.getDisplayName())
+ .explanation("a key ID must be set when client-side
encryption is enabled.").build());
+ }
+
+ if (cseKeyType == AzureBlobClientSideEncryptionMethod.SYMMETRIC) {
+ validationResults.addAll(validateSymmetricKey(cseSymmetricKeyHex));
+ }
+
+ return validationResults;
+ }
+
+ private static List<ValidationResult> validateSymmetricKey(String keyHex) {
+ List<ValidationResult> validationResults = new ArrayList<>();
+ if (Strings.isNullOrEmpty(keyHex)) {
+ validationResults.add(new
ValidationResult.Builder().subject(CSE_SYMMETRIC_KEY_HEX.getDisplayName())
+ .explanation("a symmetric key must not be set when
client-side encryption is enabled with symmetric encryption.").build());
+ } else {
+ byte[] keyBytes;
+ try {
+ keyBytes = Hex.decodeHex(keyHex.toCharArray());
+ new SymmetricKey("nifi", keyBytes);
Review comment:
```suggestion
new SymmetricKey(DEFAULT_KEY_ID, keyBytes);
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
Review comment:
These are usually in an all-caps title case.
```suggestion
.displayName("Client-Side Encryption Key Type")
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
+ .required(true)
+ .allowableValues(buildCseEncryptionMethodAllowableValues())
+ .defaultValue(AzureBlobClientSideEncryptionMethod.NONE.name())
+ .description("Specifies the key type to use for client-side
encryption.")
+ .build();
+
+ public static final PropertyDescriptor CSE_KEY_ID = new
PropertyDescriptor.Builder()
+ .name("cse-key-id")
+ .displayName("Client-Side Encryption key ID")
+ .description("Specifies the ID of the key to use for client-side
encryption.")
+
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CSE_SYMMETRIC_KEY_HEX = new
PropertyDescriptor.Builder()
+ .name("cse-symmetric-key-hex")
+ .displayName("Symmetric Key (hexadecimal)")
+ .description("When using symmetric client-side encryption, this is
the raw key, encoded in hexadecimal")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
Review comment:
```suggestion
.addValidator(StandardValidators.NON_BLANK_VALIDATOR)
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITListAzureBlobStorage.java
##########
@@ -34,11 +41,11 @@
public void setUp() throws Exception {
uploadTestBlob();
-
Thread.sleep(ListAzureBlobStorage.LISTING_LAG_MILLIS.get(TimeUnit.SECONDS));
+
Thread.sleep(ListAzureBlobStorage.LISTING_LAG_MILLIS.get(TimeUnit.SECONDS) * 2);
}
@Test
- public void testListBlobs() {
+ public void testListBlobs() throws Exception{
Review comment:
```suggestion
public void testListBlobs() throws Exception {
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
+ .required(true)
+ .allowableValues(buildCseEncryptionMethodAllowableValues())
+ .defaultValue(AzureBlobClientSideEncryptionMethod.NONE.name())
+ .description("Specifies the key type to use for client-side
encryption.")
+ .build();
+
+ public static final PropertyDescriptor CSE_KEY_ID = new
PropertyDescriptor.Builder()
+ .name("cse-key-id")
+ .displayName("Client-Side Encryption key ID")
+ .description("Specifies the ID of the key to use for client-side
encryption.")
+
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CSE_SYMMETRIC_KEY_HEX = new
PropertyDescriptor.Builder()
+ .name("cse-symmetric-key-hex")
+ .displayName("Symmetric Key (hexadecimal)")
+ .description("When using symmetric client-side encryption, this is
the raw key, encoded in hexadecimal")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(true)
+ .build();
+
+ private static AllowableValue[] buildCseEncryptionMethodAllowableValues() {
+ final AzureBlobClientSideEncryptionMethod[] encryptionMethods =
AzureBlobClientSideEncryptionMethod.values();
+ List<AllowableValue> allowableValues = new
ArrayList<>(encryptionMethods.length);
+ for (AzureBlobClientSideEncryptionMethod em : encryptionMethods) {
+ allowableValues.add(new AllowableValue(em.name(), em.name(),
em.toString()));
+ }
+
+ return allowableValues.toArray(new AllowableValue[0]);
+ }
+
+ public static Collection<ValidationResult>
validateClientSideEncryptionProperties(ValidationContext validationContext) {
+ final List<ValidationResult> validationResults = new ArrayList<>();
+
+ final String cseKeyTypeValue =
validationContext.getProperty(CSE_KEY_TYPE).getValue();
+ final AzureBlobClientSideEncryptionMethod cseKeyType =
AzureBlobClientSideEncryptionMethod.valueOf(cseKeyTypeValue);
+
+ final String cseKeyId =
validationContext.getProperty(CSE_KEY_ID).getValue();
+
+ final String cseSymmetricKeyHex =
validationContext.getProperty(CSE_SYMMETRIC_KEY_HEX).getValue();
+
+ if (cseKeyType != AzureBlobClientSideEncryptionMethod.NONE &&
Strings.isNullOrEmpty(cseKeyId)) {
+ validationResults.add(new
ValidationResult.Builder().subject(CSE_KEY_ID.getDisplayName())
+ .explanation("a key ID must be set when client-side
encryption is enabled.").build());
+ }
+
+ if (cseKeyType == AzureBlobClientSideEncryptionMethod.SYMMETRIC) {
+ validationResults.addAll(validateSymmetricKey(cseSymmetricKeyHex));
+ }
+
+ return validationResults;
+ }
+
+ private static List<ValidationResult> validateSymmetricKey(String keyHex) {
+ List<ValidationResult> validationResults = new ArrayList<>();
+ if (Strings.isNullOrEmpty(keyHex)) {
+ validationResults.add(new
ValidationResult.Builder().subject(CSE_SYMMETRIC_KEY_HEX.getDisplayName())
+ .explanation("a symmetric key must not be set when
client-side encryption is enabled with symmetric encryption.").build());
+ } else {
+ byte[] keyBytes;
+ try {
+ keyBytes = Hex.decodeHex(keyHex.toCharArray());
+ new SymmetricKey("nifi", keyBytes);
+ } catch (DecoderException e) {
+ validationResults.add(new
ValidationResult.Builder().subject(CSE_SYMMETRIC_KEY_HEX.getDisplayName())
+ .explanation("the symmetric key must be valid
hexadecimal string.").build());
Review comment:
```suggestion
.explanation("the symmetric key must be a valid
hexadecimal string.").build());
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
+ .required(true)
+ .allowableValues(buildCseEncryptionMethodAllowableValues())
+ .defaultValue(AzureBlobClientSideEncryptionMethod.NONE.name())
+ .description("Specifies the key type to use for client-side
encryption.")
+ .build();
+
+ public static final PropertyDescriptor CSE_KEY_ID = new
PropertyDescriptor.Builder()
+ .name("cse-key-id")
+ .displayName("Client-Side Encryption key ID")
+ .description("Specifies the ID of the key to use for client-side
encryption.")
+
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CSE_SYMMETRIC_KEY_HEX = new
PropertyDescriptor.Builder()
+ .name("cse-symmetric-key-hex")
+ .displayName("Symmetric Key (hexadecimal)")
+ .description("When using symmetric client-side encryption, this is
the raw key, encoded in hexadecimal")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(true)
+ .build();
+
+ private static AllowableValue[] buildCseEncryptionMethodAllowableValues() {
+ final AzureBlobClientSideEncryptionMethod[] encryptionMethods =
AzureBlobClientSideEncryptionMethod.values();
+ List<AllowableValue> allowableValues = new
ArrayList<>(encryptionMethods.length);
+ for (AzureBlobClientSideEncryptionMethod em : encryptionMethods) {
+ allowableValues.add(new AllowableValue(em.name(), em.name(),
em.toString()));
+ }
+
+ return allowableValues.toArray(new AllowableValue[0]);
+ }
+
+ public static Collection<ValidationResult>
validateClientSideEncryptionProperties(ValidationContext validationContext) {
+ final List<ValidationResult> validationResults = new ArrayList<>();
+
+ final String cseKeyTypeValue =
validationContext.getProperty(CSE_KEY_TYPE).getValue();
+ final AzureBlobClientSideEncryptionMethod cseKeyType =
AzureBlobClientSideEncryptionMethod.valueOf(cseKeyTypeValue);
+
+ final String cseKeyId =
validationContext.getProperty(CSE_KEY_ID).getValue();
+
+ final String cseSymmetricKeyHex =
validationContext.getProperty(CSE_SYMMETRIC_KEY_HEX).getValue();
+
+ if (cseKeyType != AzureBlobClientSideEncryptionMethod.NONE &&
Strings.isNullOrEmpty(cseKeyId)) {
Review comment:
```suggestion
if (cseKeyType != AzureBlobClientSideEncryptionMethod.NONE &&
StringUtils.isBlank(cseKeyId)) {
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
+ .required(true)
+ .allowableValues(buildCseEncryptionMethodAllowableValues())
+ .defaultValue(AzureBlobClientSideEncryptionMethod.NONE.name())
+ .description("Specifies the key type to use for client-side
encryption.")
+ .build();
+
+ public static final PropertyDescriptor CSE_KEY_ID = new
PropertyDescriptor.Builder()
+ .name("cse-key-id")
+ .displayName("Client-Side Encryption key ID")
Review comment:
```suggestion
.displayName("Client-Side Encryption Key ID")
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionMethod.java
##########
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+/**
+ * Enumeration capturing essential information about the various client-side
+ * encryption methods supported by Azure
+ */
+public enum AzureBlobClientSideEncryptionMethod {
+
+ NONE("None", "The blobs sent to Azure is not encrypted."),
+ SYMMETRIC("Symmetric","The blobs sent to Azure are encrypted using a
symmetric algorithm.");
Review comment:
```suggestion
NONE("None", "The blobs sent to Azure are not encrypted."),
SYMMETRIC("Symmetric", "The blobs sent to Azure are encrypted using a
symmetric algorithm.");
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
+ .required(true)
+ .allowableValues(buildCseEncryptionMethodAllowableValues())
+ .defaultValue(AzureBlobClientSideEncryptionMethod.NONE.name())
+ .description("Specifies the key type to use for client-side
encryption.")
+ .build();
+
+ public static final PropertyDescriptor CSE_KEY_ID = new
PropertyDescriptor.Builder()
+ .name("cse-key-id")
+ .displayName("Client-Side Encryption key ID")
+ .description("Specifies the ID of the key to use for client-side
encryption.")
+
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CSE_SYMMETRIC_KEY_HEX = new
PropertyDescriptor.Builder()
+ .name("cse-symmetric-key-hex")
+ .displayName("Symmetric Key (hexadecimal)")
Review comment:
```suggestion
.displayName("Symmetric Key")
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/main/java/org/apache/nifi/processors/azure/storage/utils/AzureBlobClientSideEncryptionUtils.java
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage.utils;
+
+import com.google.common.base.Strings;
+import com.microsoft.azure.keyvault.cryptography.SymmetricKey;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.nifi.components.AllowableValue;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.expression.ExpressionLanguageScope;
+import org.apache.nifi.processor.util.StandardValidators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class AzureBlobClientSideEncryptionUtils {
+ public static final PropertyDescriptor CSE_KEY_TYPE = new
PropertyDescriptor.Builder()
+ .name("cse-key-type")
+ .displayName("Client-Side Encryption key type")
+ .required(true)
+ .allowableValues(buildCseEncryptionMethodAllowableValues())
+ .defaultValue(AzureBlobClientSideEncryptionMethod.NONE.name())
+ .description("Specifies the key type to use for client-side
encryption.")
+ .build();
+
+ public static final PropertyDescriptor CSE_KEY_ID = new
PropertyDescriptor.Builder()
+ .name("cse-key-id")
+ .displayName("Client-Side Encryption key ID")
+ .description("Specifies the ID of the key to use for client-side
encryption.")
+
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .build();
+
+ public static final PropertyDescriptor CSE_SYMMETRIC_KEY_HEX = new
PropertyDescriptor.Builder()
+ .name("cse-symmetric-key-hex")
+ .displayName("Symmetric Key (hexadecimal)")
+ .description("When using symmetric client-side encryption, this is
the raw key, encoded in hexadecimal")
+ .required(false)
+ .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
+ .sensitive(true)
+ .build();
+
+ private static AllowableValue[] buildCseEncryptionMethodAllowableValues() {
+ final AzureBlobClientSideEncryptionMethod[] encryptionMethods =
AzureBlobClientSideEncryptionMethod.values();
+ List<AllowableValue> allowableValues = new
ArrayList<>(encryptionMethods.length);
+ for (AzureBlobClientSideEncryptionMethod em : encryptionMethods) {
+ allowableValues.add(new AllowableValue(em.name(), em.name(),
em.toString()));
+ }
+
+ return allowableValues.toArray(new AllowableValue[0]);
+ }
+
+ public static Collection<ValidationResult>
validateClientSideEncryptionProperties(ValidationContext validationContext) {
+ final List<ValidationResult> validationResults = new ArrayList<>();
+
+ final String cseKeyTypeValue =
validationContext.getProperty(CSE_KEY_TYPE).getValue();
+ final AzureBlobClientSideEncryptionMethod cseKeyType =
AzureBlobClientSideEncryptionMethod.valueOf(cseKeyTypeValue);
+
+ final String cseKeyId =
validationContext.getProperty(CSE_KEY_ID).getValue();
+
+ final String cseSymmetricKeyHex =
validationContext.getProperty(CSE_SYMMETRIC_KEY_HEX).getValue();
+
+ if (cseKeyType != AzureBlobClientSideEncryptionMethod.NONE &&
Strings.isNullOrEmpty(cseKeyId)) {
+ validationResults.add(new
ValidationResult.Builder().subject(CSE_KEY_ID.getDisplayName())
+ .explanation("a key ID must be set when client-side
encryption is enabled.").build());
+ }
+
+ if (cseKeyType == AzureBlobClientSideEncryptionMethod.SYMMETRIC) {
+ validationResults.addAll(validateSymmetricKey(cseSymmetricKeyHex));
+ }
+
+ return validationResults;
+ }
+
+ private static List<ValidationResult> validateSymmetricKey(String keyHex) {
+ List<ValidationResult> validationResults = new ArrayList<>();
Review comment:
```suggestion
final List<ValidationResult> validationResults = new ArrayList<>();
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureBlobStorage.java
##########
@@ -48,6 +59,74 @@ public void testPutBlob() throws Exception {
assertResult();
}
+ @Test
+ public void testPutBlob64BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_64B_VALUE);
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testPutBlob128BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_128B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
+ runner.run();
+
+ assertResult();
+ }
+
+ @Test
+ public void testPutBlob192BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_192B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
+ runner.run();
+
+ assertResult();
+ }
+
+ @Test
+ public void testPutBlob256BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_256B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
+ runner.run();
+
+ assertResult();
+ }
+
+ @Test
+ public void testPutBlob384BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_384B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
Review comment:
```suggestion
runner.enqueue(TEST_FILE_CONTENT.getBytes());
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureBlobStorage.java
##########
@@ -48,6 +59,74 @@ public void testPutBlob() throws Exception {
assertResult();
}
+ @Test
+ public void testPutBlob64BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_64B_VALUE);
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testPutBlob128BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_128B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
+ runner.run();
+
+ assertResult();
+ }
+
+ @Test
+ public void testPutBlob192BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_192B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
+ runner.run();
+
+ assertResult();
+ }
+
+ @Test
+ public void testPutBlob256BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_256B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
Review comment:
```suggestion
runner.enqueue(TEST_FILE_CONTENT.getBytes());
```
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITAzureBlobStorageE2E.java
##########
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.azure.storage;
+
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.StorageCredentials;
+import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
+import com.microsoft.azure.storage.blob.CloudBlobClient;
+import com.microsoft.azure.storage.blob.CloudBlobContainer;
+import org.apache.nifi.processors.azure.AbstractAzureBlobProcessor;
+import
org.apache.nifi.processors.azure.storage.utils.AzureBlobClientSideEncryptionMethod;
+import
org.apache.nifi.processors.azure.storage.utils.AzureBlobClientSideEncryptionUtils;
+import org.apache.nifi.processors.azure.storage.utils.AzureStorageUtils;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.apache.nifi.util.file.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ComparisonFailure;
+import org.junit.Test;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.fail;
+
+public class ITAzureBlobStorageE2E {
Review comment:
Should any of this be moved to the `AbstractAzureBlobStorageIT`? (In
which case, already existing tests would need to be updated too.)
##########
File path:
nifi-nar-bundles/nifi-azure-bundle/nifi-azure-processors/src/test/java/org/apache/nifi/processors/azure/storage/ITPutAzureBlobStorage.java
##########
@@ -48,6 +59,74 @@ public void testPutBlob() throws Exception {
assertResult();
}
+ @Test
+ public void testPutBlob64BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_64B_VALUE);
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testPutBlob128BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_128B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
+ runner.run();
+
+ assertResult();
+ }
+
+ @Test
+ public void testPutBlob192BSymmetricCSE() throws Exception {
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_TYPE,
AzureBlobClientSideEncryptionMethod.SYMMETRIC.name());
+ runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_KEY_ID,
KEY_ID_VALUE);
+
runner.setProperty(AzureBlobClientSideEncryptionUtils.CSE_SYMMETRIC_KEY_HEX,
KEY_192B_VALUE);
+ runner.assertValid();
+ runner.enqueue("0123456789".getBytes());
Review comment:
```suggestion
runner.enqueue(TEST_FILE_CONTENT.getBytes());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
