This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 6689e45364 HDDS-9028. Migrate TestOzoneAtRestEncryption to JUnit5
(#5079)
6689e45364 is described below
commit 6689e453642d2ab1b613ed8f2691e2c6a5e5deb5
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Jul 19 07:01:47 2023 +0200
HDDS-9028. Migrate TestOzoneAtRestEncryption to JUnit5 (#5079)
---
.../client/rpc/TestOzoneAtRestEncryption.java | 247 ++++++++++-----------
1 file changed, 123 insertions(+), 124 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
index c6698c663c..c40b46a79d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java
@@ -32,7 +32,6 @@ import java.util.Map;
import java.util.Random;
import java.util.TreeMap;
import java.util.UUID;
-import java.util.Collection;
import com.google.common.cache.Cache;
import org.apache.hadoop.conf.StorageUnit;
@@ -79,27 +78,23 @@ import org.apache.ozone.test.GenericTestUtils;
import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.client.ReplicationType.RATIS;
-
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.ozone.test.tag.Flaky;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
import org.mockito.Mockito;
-/**
- * This class is to test all the public facing APIs of Ozone Client.
- */
-@RunWith(Parameterized.class)
-public class TestOzoneAtRestEncryption {
-
- @Parameterized.Parameters
- public static Collection<BucketLayout> data() {
- return Arrays.asList(
- BucketLayout.FILE_SYSTEM_OPTIMIZED,
- BucketLayout.OBJECT_STORE);
- }
+class TestOzoneAtRestEncryption {
private static MiniOzoneCluster cluster = null;
private static MiniKMS miniKMS;
@@ -122,19 +117,14 @@ public class TestOzoneAtRestEncryption {
private static final int DEFAULT_CRYPTO_BUFFER_SIZE = 8 * 1024; // 8KB
// (this is the default Crypto Buffer size as determined by the config
// hadoop.security.crypto.buffer.size)
- private final BucketLayout bucketLayout;
-
- public TestOzoneAtRestEncryption(BucketLayout layout) {
- bucketLayout = layout;
- }
- @BeforeClass
- public static void init() throws Exception {
+ @BeforeAll
+ static void init() throws Exception {
testDir = GenericTestUtils.getTestDir(
TestSecureOzoneRpcClient.class.getSimpleName());
File kmsDir = new File(testDir, UUID.randomUUID().toString());
- Assert.assertTrue(kmsDir.mkdirs());
+ assertTrue(kmsDir.mkdirs());
MiniKMS.Builder miniKMSBuilder = new MiniKMS.Builder();
miniKMS = miniKMSBuilder.setKmsConfDir(kmsDir).build();
miniKMS.start();
@@ -178,11 +168,8 @@ public class TestOzoneAtRestEncryption {
createKey(TEST_KEY, cluster.getOzoneManager().getKmsProvider(), conf);
}
- /**
- * Close OzoneClient and shutdown MiniOzoneCluster.
- */
- @AfterClass
- public static void shutdown() throws IOException {
+ @AfterAll
+ static void shutdown() throws IOException {
if (ozClient != null) {
ozClient.close();
}
@@ -200,8 +187,9 @@ public class TestOzoneAtRestEncryption {
}
}
- @Test
- public void testPutKeyWithEncryption() throws Exception {
+ @ParameterizedTest
+ @EnumSource
+ void testPutKeyWithEncryption(BucketLayout bucketLayout) throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
@@ -217,13 +205,14 @@ public class TestOzoneAtRestEncryption {
createAndVerifyStreamKeyData(bucket);
}
- @Test
- public void testLinkEncryptedBuckets() throws Exception {
+ @ParameterizedTest
+ @EnumSource
+ void testLinkEncryptedBuckets(BucketLayout bucketLayout) throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
// Create source volume/bucket.
- createVolumeAndBucket(volumeName, bucketName);
+ createVolumeAndBucket(volumeName, bucketName, bucketLayout);
// Create link volume/bucket.
String linkVolumeName = UUID.randomUUID().toString();
@@ -266,11 +255,11 @@ public class TestOzoneAtRestEncryption {
Instant testStartTime) throws Exception {
// Verify content.
OzoneKeyDetails key = bucket.getKey(keyName);
- Assert.assertEquals(keyName, key.getName());
+ assertEquals(keyName, key.getName());
// Check file encryption info is set,
// if set key will use this encryption info and encrypt data.
- Assert.assertNotNull(key.getFileEncryptionInfo());
+ assertNotNull(key.getFileEncryptionInfo());
byte[] fileContent;
int len = 0;
@@ -281,17 +270,17 @@ public class TestOzoneAtRestEncryption {
}
- Assert.assertEquals(len, value.length());
- Assert.assertTrue(verifyRatisReplication(bucket.getVolumeName(),
- bucket.getName(), keyName, ReplicationType.RATIS,
- ReplicationFactor.ONE));
- Assert.assertEquals(value, new String(fileContent,
StandardCharsets.UTF_8));
- Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
- Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
+ assertEquals(len, value.length());
+ assertTrue(verifyRatisReplication(bucket.getVolumeName(),
+ bucket.getName(), keyName, RATIS,
+ ONE));
+ assertEquals(value, new String(fileContent, StandardCharsets.UTF_8));
+ assertFalse(key.getCreationTime().isBefore(testStartTime));
+ assertFalse(key.getModificationTime().isBefore(testStartTime));
}
private OzoneBucket createVolumeAndBucket(String volumeName,
- String bucketName) throws Exception {
+ String bucketName, BucketLayout bucketLayout) throws Exception {
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
BucketArgs bucketArgs = BucketArgs.newBuilder()
@@ -306,8 +295,9 @@ public class TestOzoneAtRestEncryption {
store.createVolume(linkVol);
OzoneVolume linkVolume = store.getVolume(linkVol);
BucketArgs linkBucketArgs = BucketArgs.newBuilder()
- .setSourceVolume(sourceVol).setSourceBucket(sourceBucket)
- .setBucketLayout(bucketLayout).build();
+ .setSourceVolume(sourceVol)
+ .setSourceBucket(sourceBucket)
+ .build();
linkVolume.createBucket(linkBucket, linkBucketArgs);
return linkVolume.getBucket(linkBucket);
}
@@ -317,10 +307,11 @@ public class TestOzoneAtRestEncryption {
* 1. Create a GDPR enforced bucket
* 2. PutKey with Encryption in above bucket and verify.
* 3. DeleteKey and confirm the metadata does not have encryption key.
- * @throws Exception
*/
- @Test
- public void testKeyWithEncryptionAndGdpr() throws Exception {
+ @ParameterizedTest
+ @EnumSource
+ void testKeyWithEncryptionAndGdpr(BucketLayout bucketLayout)
+ throws Exception {
//Step 1
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
@@ -336,10 +327,9 @@ public class TestOzoneAtRestEncryption {
.setBucketLayout(bucketLayout).build();
volume.createBucket(bucketName, bucketArgs);
OzoneBucket bucket = volume.getBucket(bucketName);
- Assert.assertEquals(bucketName, bucket.getName());
- Assert.assertNotNull(bucket.getMetadata());
- Assert.assertEquals("true",
- bucket.getMetadata().get(OzoneConsts.GDPR_FLAG));
+ assertEquals(bucketName, bucket.getName());
+ assertNotNull(bucket.getMetadata());
+ assertEquals("true", bucket.getMetadata().get(OzoneConsts.GDPR_FLAG));
//Step 2
String keyName = UUID.randomUUID().toString();
@@ -353,7 +343,7 @@ public class TestOzoneAtRestEncryption {
}
OzoneKeyDetails key = bucket.getKey(keyName);
- Assert.assertEquals(keyName, key.getName());
+ assertEquals(keyName, key.getName());
byte[] fileContent;
int len = 0;
@@ -362,16 +352,16 @@ public class TestOzoneAtRestEncryption {
len = is.read(fileContent);
}
- Assert.assertEquals(len, value.length());
- Assert.assertTrue(verifyRatisReplication(volumeName, bucketName,
- keyName, ReplicationType.RATIS,
- ReplicationFactor.ONE));
- Assert.assertEquals(value, new String(fileContent,
StandardCharsets.UTF_8));
- Assert.assertFalse(key.getCreationTime().isBefore(testStartTime));
- Assert.assertFalse(key.getModificationTime().isBefore(testStartTime));
- Assert.assertEquals("true", key.getMetadata().get(OzoneConsts.GDPR_FLAG));
+ assertEquals(len, value.length());
+ assertTrue(verifyRatisReplication(volumeName, bucketName,
+ keyName, RATIS,
+ ONE));
+ assertEquals(value, new String(fileContent, StandardCharsets.UTF_8));
+ assertFalse(key.getCreationTime().isBefore(testStartTime));
+ assertFalse(key.getModificationTime().isBefore(testStartTime));
+ assertEquals("true", key.getMetadata().get(OzoneConsts.GDPR_FLAG));
//As TDE is enabled, the TDE encryption details should not be null.
- Assert.assertNotNull(key.getFileEncryptionInfo());
+ assertNotNull(key.getFileEncryptionInfo());
//Step 3
bucket.deleteKey(key.getName());
@@ -387,15 +377,13 @@ public class TestOzoneAtRestEncryption {
}, 500, 100000);
RepeatedOmKeyInfo deletedKeys =
getMatchedKeyInfo(keyName, omMetadataManager);
- Assert.assertNotNull(deletedKeys);
+ assertNotNull(deletedKeys);
Map<String, String> deletedKeyMetadata =
deletedKeys.getOmKeyInfoList().get(0).getMetadata();
- Assert.assertFalse(deletedKeyMetadata.containsKey(OzoneConsts.GDPR_FLAG));
-
Assert.assertFalse(deletedKeyMetadata.containsKey(OzoneConsts.GDPR_SECRET));
- Assert.assertFalse(
- deletedKeyMetadata.containsKey(OzoneConsts.GDPR_ALGORITHM));
- Assert.assertNull(
- deletedKeys.getOmKeyInfoList().get(0).getFileEncryptionInfo());
+ assertFalse(deletedKeyMetadata.containsKey(OzoneConsts.GDPR_FLAG));
+ assertFalse(deletedKeyMetadata.containsKey(OzoneConsts.GDPR_SECRET));
+ assertFalse(deletedKeyMetadata.containsKey(OzoneConsts.GDPR_ALGORITHM));
+ assertNull(deletedKeys.getOmKeyInfoList().get(0).getFileEncryptionInfo());
}
static boolean verifyRatisReplication(String volumeName, String bucketName,
@@ -439,65 +427,77 @@ public class TestOzoneAtRestEncryption {
provider.flush();
}
- @Test
- public void testMPUwithOnePart() throws Exception {
+ @ParameterizedTest
+ @EnumSource
+ void mpuOnePart(BucketLayout bucketLayout) throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
- OzoneBucket bucket = createVolumeAndBucket(volumeName, bucketName);
- testMultipartUploadWithEncryption(bucket, 1);
+ testMultipartUploadWithEncryption(
+ createVolumeAndBucket(volumeName, bucketName, bucketLayout), 1);
}
- @Test
- public void testMPUwithTwoParts() throws Exception {
+ @ParameterizedTest
+ @EnumSource
+ void mpuTwoParts(BucketLayout bucketLayout) throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
- OzoneBucket bucket = createVolumeAndBucket(volumeName, bucketName);
- testMultipartUploadWithEncryption(bucket, 2);
+ testMultipartUploadWithEncryption(
+ createVolumeAndBucket(volumeName, bucketName, bucketLayout), 2);
}
- @Test
- public void testMPUwithThreePartsOverride() throws Exception {
+ @ParameterizedTest
+ @EnumSource
+ void mpuThreePartsOverride(BucketLayout bucketLayout) throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
- OzoneBucket bucket = createVolumeAndBucket(volumeName, bucketName);
+ OzoneBucket bucket = createVolumeAndBucket(volumeName, bucketName,
+ bucketLayout);
testMultipartUploadWithEncryption(bucket, 3);
// override the key and check content
testMultipartUploadWithEncryption(bucket, 3);
}
- @Test
- public void testMPUwithOneStreamPart() throws Exception {
+ @ParameterizedTest
+ @EnumSource
+ @Flaky("HDDS-8947")
+ void mpuStreamOnePart(BucketLayout bucketLayout) throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
- OzoneBucket bucket = createVolumeAndBucket(volumeName, bucketName);
- testMultipartUploadWithEncryption(bucket, 1, true);
+ testMultipartUploadWithEncryption(
+ createVolumeAndBucket(volumeName, bucketName, bucketLayout), 1, true);
}
- @Test
- public void testMPUwithTwoStreamParts() throws Exception {
+ @ParameterizedTest
+ @EnumSource
+ @Flaky("HDDS-8947")
+ void mpuStreamTwoParts(BucketLayout bucketLayout) throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
- OzoneBucket bucket = createVolumeAndBucket(volumeName, bucketName);
- testMultipartUploadWithEncryption(bucket, 2, true);
+ testMultipartUploadWithEncryption(
+ createVolumeAndBucket(volumeName, bucketName, bucketLayout), 2, true);
}
- @Test
- public void testMPUwithThreeStreamPartsOverride() throws Exception {
+ @ParameterizedTest
+ @EnumSource
+ @Flaky("HDDS-8947")
+ void mpuStreamThreePartsOverride(BucketLayout bucketLayout) throws Exception
{
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
- OzoneBucket bucket = createVolumeAndBucket(volumeName, bucketName);
+ OzoneBucket bucket = createVolumeAndBucket(volumeName, bucketName,
+ bucketLayout);
testMultipartUploadWithEncryption(bucket, 3);
// override the key and check content
testMultipartUploadWithEncryption(bucket, 3, true);
}
- @Test
- public void testMPUwithLinkBucket() throws Exception {
+ @ParameterizedTest
+ @EnumSource
+ void mpuWithLink(BucketLayout bucketLayout) throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
- createVolumeAndBucket(volumeName, bucketName);
+ createVolumeAndBucket(volumeName, bucketName, bucketLayout);
String linkVolumeName = UUID.randomUUID().toString();
String linkBucketName = UUID.randomUUID().toString();
@@ -506,12 +506,12 @@ public class TestOzoneAtRestEncryption {
testMultipartUploadWithEncryption(linkBucket, 2);
}
- public void testMultipartUploadWithEncryption(OzoneBucket bucket,
+ private void testMultipartUploadWithEncryption(OzoneBucket bucket,
int numParts) throws Exception {
testMultipartUploadWithEncryption(bucket, numParts, false);
}
- public void testMultipartUploadWithEncryption(OzoneBucket bucket,
+ private void testMultipartUploadWithEncryption(OzoneBucket bucket,
int numParts, boolean isStream) throws Exception {
String keyName = "mpu_test_key_" + numParts;
@@ -559,14 +559,14 @@ public class TestOzoneAtRestEncryption {
// Create an input stream to read the data
try (OzoneInputStream inputStream = bucket.readKey(keyName)) {
- Assert.assertTrue(inputStream.getInputStream()
+ assertTrue(inputStream.getInputStream()
instanceof MultipartInputStream);
// Test complete read
byte[] completeRead = new byte[keySize];
int bytesRead = inputStream.read(completeRead, 0, keySize);
- Assert.assertEquals(bytesRead, keySize);
- Assert.assertArrayEquals(inputData, completeRead);
+ assertEquals(bytesRead, keySize);
+ assertArrayEquals(inputData, completeRead);
// Read different data lengths and starting from different offsets and
// verify the data matches.
@@ -595,9 +595,8 @@ public class TestOzoneAtRestEncryption {
int actualReadLen = inputStream.read(readData, 0, readDataLen);
assertReadContent(inputData, readData, readFromPosition);
- Assert.assertEquals(readFromPosition + readDataLen,
- inputStream.getPos());
- Assert.assertEquals(readDataLen, actualReadLen);
+ assertEquals(readFromPosition + readDataLen, inputStream.getPos());
+ assertEquals(readDataLen, actualReadLen);
}
}
}
@@ -616,7 +615,7 @@ public class TestOzoneAtRestEncryption {
replicationConfig);
String uploadID = multipartInfo.getUploadID();
- Assert.assertNotNull(uploadID);
+ assertNotNull(uploadID);
return uploadID;
}
@@ -635,8 +634,8 @@ public class TestOzoneAtRestEncryption {
OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo =
multipartStreamKey.getCommitUploadPartInfo();
- Assert.assertNotNull(omMultipartCommitUploadPartInfo);
- Assert.assertNotNull(omMultipartCommitUploadPartInfo.getPartName());
+ assertNotNull(omMultipartCommitUploadPartInfo);
+ assertNotNull(omMultipartCommitUploadPartInfo.getPartName());
return omMultipartCommitUploadPartInfo.getPartName();
}
@@ -650,8 +649,8 @@ public class TestOzoneAtRestEncryption {
OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo =
ozoneOutputStream.getCommitUploadPartInfo();
- Assert.assertNotNull(omMultipartCommitUploadPartInfo);
- Assert.assertNotNull(omMultipartCommitUploadPartInfo.getPartName());
+ assertNotNull(omMultipartCommitUploadPartInfo);
+ assertNotNull(omMultipartCommitUploadPartInfo.getPartName());
return omMultipartCommitUploadPartInfo.getPartName();
}
@@ -660,41 +659,41 @@ public class TestOzoneAtRestEncryption {
OmMultipartUploadCompleteInfo omMultipartUploadCompleteInfo = bucket
.completeMultipartUpload(keyName, uploadID, partsMap);
- Assert.assertNotNull(omMultipartUploadCompleteInfo);
- Assert.assertEquals(omMultipartUploadCompleteInfo.getBucket(), bucket
+ assertNotNull(omMultipartUploadCompleteInfo);
+ assertEquals(omMultipartUploadCompleteInfo.getBucket(), bucket
.getName());
- Assert.assertEquals(omMultipartUploadCompleteInfo.getVolume(), bucket
+ assertEquals(omMultipartUploadCompleteInfo.getVolume(), bucket
.getVolumeName());
- Assert.assertEquals(omMultipartUploadCompleteInfo.getKey(), keyName);
- Assert.assertNotNull(omMultipartUploadCompleteInfo.getHash());
+ assertEquals(omMultipartUploadCompleteInfo.getKey(), keyName);
+ assertNotNull(omMultipartUploadCompleteInfo.getHash());
}
private static void assertReadContent(byte[] inputData, byte[] readData,
int offset) {
byte[] inputDataForComparison = Arrays.copyOfRange(inputData, offset,
offset + readData.length);
- Assert.assertArrayEquals("Read data does not match input data at offset " +
- offset + " and length " + readData.length,
- inputDataForComparison, readData);
+ assertArrayEquals(inputDataForComparison, readData,
+ "Read data does not match input data at offset " +
+ offset + " and length " + readData.length);
}
@Test
- public void testGetKeyProvider() throws Exception {
+ void testGetKeyProvider() throws Exception {
KeyProvider kp1 = store.getKeyProvider();
KeyProvider kpSpy = Mockito.spy(kp1);
- Assert.assertNotEquals(kpSpy, kp1);
+ assertNotEquals(kpSpy, kp1);
Cache<URI, KeyProvider> cacheSpy =
((RpcClient)store.getClientProxy()).getKeyProviderCache();
cacheSpy.put(store.getKeyProviderUri(), kpSpy);
KeyProvider kp2 = store.getKeyProvider();
- Assert.assertEquals(kpSpy, kp2);
+ assertEquals(kpSpy, kp2);
// Verify the spied key provider is closed upon ozone client close
ozClient.close();
Mockito.verify(kpSpy).close();
KeyProvider kp3 = ozClient.getObjectStore().getKeyProvider();
- Assert.assertNotEquals(kp3, kpSpy);
+ assertNotEquals(kp3, kpSpy);
// Restore ozClient and store
TestOzoneRpcClient.setOzClient(OzoneClientFactory.getRpcClient(conf));
TestOzoneRpcClient.setStore(ozClient.getObjectStore());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
