This is an automated email from the ASF dual-hosted git repository.
rohangarg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new ae4ea51352b Rewrite S3StorageConnectorTest using testcontainers and
MinIO (#17539)
ae4ea51352b is described below
commit ae4ea51352bc6d0a6b9d6078478b516aee36a073
Author: Rohan Garg <[email protected]>
AuthorDate: Mon Dec 9 09:48:38 2024 -0500
Rewrite S3StorageConnectorTest using testcontainers and MinIO (#17539)
---
.github/scripts/unit_tests_script.sh | 2 +-
extensions-core/s3-extensions/pom.xml | 44 ++
.../storage/s3/ServerSideEncryptingAmazonS3.java | 5 +
.../storage/s3/output/S3StorageConnector.java | 4 +-
.../storage/s3/output/S3StorageConnectorTest.java | 449 +++++++++++----------
pom.xml | 15 +
6 files changed, 303 insertions(+), 216 deletions(-)
diff --git a/.github/scripts/unit_tests_script.sh
b/.github/scripts/unit_tests_script.sh
index e048a19c4af..28de3e94379 100755
--- a/.github/scripts/unit_tests_script.sh
+++ b/.github/scripts/unit_tests_script.sh
@@ -22,7 +22,7 @@ unset _JAVA_OPTIONS
# Set MAVEN_OPTS for Surefire launcher.
MAVEN_OPTS='-Xmx2500m' ${MVN} test -pl ${MAVEN_PROJECTS} \
${MAVEN_SKIP}
-Ddruid.generic.useDefaultValueForNull=${DRUID_USE_DEFAULT_VALUE_FOR_NULL} \
--DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}"
+-DjfrProfilerArgLine="${JFR_PROFILER_ARG_LINE}" -Pci
sh -c "dmesg | egrep -i '(oom|out of memory|kill process|killed).*' -C 1 ||
exit 0"
free -m
${MVN} -pl ${MAVEN_PROJECTS} jacoco:report || { echo "coverage_failure=false"
>> "$GITHUB_ENV" && false; }
diff --git a/extensions-core/s3-extensions/pom.xml
b/extensions-core/s3-extensions/pom.xml
index b50d9c01a9e..cb9a03a5bae 100644
--- a/extensions-core/s3-extensions/pom.xml
+++ b/extensions-core/s3-extensions/pom.xml
@@ -159,6 +159,50 @@
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>minio</artifactId>
+ <version>1.19.5</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
+ <profiles>
+ <profile>
+ <id>local</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <groups>!requires-dockerd</groups>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>ci</id>
+ </profile>
+ </profiles>
+
</project>
diff --git
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java
index 0fb93a20833..31120ba883c 100644
---
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java
+++
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java
@@ -72,6 +72,11 @@ public class ServerSideEncryptingAmazonS3
this.serverSideEncryption = serverSideEncryption;
}
+ public AmazonS3 getAmazonS3()
+ {
+ return amazonS3;
+ }
+
public boolean doesObjectExist(String bucket, String objectName)
{
try {
diff --git
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/output/S3StorageConnector.java
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/output/S3StorageConnector.java
index 8eb391a24d3..9ea0e7a8ae0 100644
---
a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/output/S3StorageConnector.java
+++
b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/output/S3StorageConnector.java
@@ -57,8 +57,8 @@ public class S3StorageConnector extends
ChunkingStorageConnector<GetObjectReques
private final ServerSideEncryptingAmazonS3 s3Client;
private final S3UploadManager s3UploadManager;
- private static final String DELIM = "/";
- private static final Joiner JOINER = Joiner.on(DELIM).skipNulls();
+ static final String DELIM = "/";
+ static final Joiner JOINER = Joiner.on(DELIM).skipNulls();
private static final int MAX_NUMBER_OF_LISTINGS = 1000;
public S3StorageConnector(S3OutputConfig config,
ServerSideEncryptingAmazonS3 serverSideEncryptingAmazonS3, S3UploadManager
s3UploadManager)
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java
index 68eaca1c42a..56df24d2115 100644
---
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java
@@ -19,160 +19,147 @@
package org.apache.druid.storage.s3.output;
+import com.amazonaws.ClientConfigurationFactory;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.internal.StaticCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AmazonS3Exception;
-import com.amazonaws.services.s3.model.DeleteObjectsRequest;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.ListObjectsV2Request;
-import com.amazonaws.services.s3.model.ListObjectsV2Result;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.druid.java.util.common.HumanReadableBytes;
+import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.metrics.StubServiceEmitter;
import org.apache.druid.query.DruidProcessingConfigTest;
import org.apache.druid.storage.StorageConnector;
-import org.apache.druid.storage.s3.NoopServerSideEncryption;
import org.apache.druid.storage.s3.ServerSideEncryptingAmazonS3;
-import org.easymock.Capture;
-import org.easymock.EasyMock;
-import org.hamcrest.CoreMatchers;
-import org.hamcrest.MatcherAssert;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.internal.matchers.ThrowableCauseMatcher;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.testcontainers.containers.MinIOContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.DockerImageName;
+
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
+import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
-import java.util.Collections;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.List;
-import java.util.stream.Collectors;
+import java.util.UUID;
+import static org.apache.druid.storage.s3.output.S3StorageConnector.JOINER;
+
+@Testcontainers
+@Tag("requires-dockerd")
public class S3StorageConnectorTest
{
- private static final String BUCKET = "BUCKET";
+ private static final String BUCKET = "testbucket";
private static final String PREFIX = "P/R/E/F/I/X";
public static final String TEST_FILE = "test.csv";
-
- private final AmazonS3Client s3Client =
EasyMock.createMock(AmazonS3Client.class);
- private final ServerSideEncryptingAmazonS3 service = new
ServerSideEncryptingAmazonS3(
- s3Client,
- new NoopServerSideEncryption()
- );
- private final ListObjectsV2Result testResult =
EasyMock.createMock(ListObjectsV2Result.class);
-
- @Rule
- public TemporaryFolder temporaryFolder = new TemporaryFolder();
+ @Container
+ private static final MinIOContainer MINIO = MinioUtil.createContainer();
+ @TempDir
+ public static File temporaryFolder;
+ private ServerSideEncryptingAmazonS3 s3Client;
private StorageConnector storageConnector;
- @Before
- public void setup()
+ @BeforeEach
+ public void setup() throws IOException
{
- try {
- S3OutputConfig s3OutputConfig = new S3OutputConfig(
- BUCKET,
- PREFIX,
- temporaryFolder.newFolder(),
- null,
- null,
- true
- );
- storageConnector = new S3StorageConnector(s3OutputConfig, service, new
S3UploadManager(
- s3OutputConfig,
- new S3ExportConfig("tempDir", new HumanReadableBytes("5MiB"), 1,
null),
- new DruidProcessingConfigTest.MockRuntimeInfo(10, 0, 0),
- new StubServiceEmitter()));
- }
- catch (IOException e) {
- throw new RuntimeException(e);
+ s3Client = MinioUtil.createS3Client(MINIO);
+ if (!s3Client.getAmazonS3().doesBucketExistV2(BUCKET)) {
+ s3Client.getAmazonS3().createBucket(new CreateBucketRequest(BUCKET));
}
+
+ S3OutputConfig s3OutputConfig = new S3OutputConfig(
+ BUCKET,
+ PREFIX,
+ Files.createDirectory(Paths.get(temporaryFolder.getAbsolutePath(),
UUID.randomUUID().toString())).toFile(),
+ null,
+ null,
+ true
+ );
+ storageConnector = new S3StorageConnector(
+ s3OutputConfig,
+ s3Client,
+ new S3UploadManager(
+ s3OutputConfig,
+ new S3ExportConfig("tempDir", new HumanReadableBytes("5MiB"), 1,
null),
+ new DruidProcessingConfigTest.MockRuntimeInfo(10, 0, 0),
+ new StubServiceEmitter()
+ )
+ );
}
@Test
public void pathExists_yes() throws IOException
{
- final Capture<GetObjectMetadataRequest> request = Capture.newInstance();
- EasyMock.reset(s3Client);
- EasyMock.expect(s3Client.getObjectMetadata(EasyMock.capture(request)))
- .andReturn(new ObjectMetadata());
- EasyMock.replay(s3Client);
- Assert.assertTrue(storageConnector.pathExists(TEST_FILE));
- Assert.assertEquals(BUCKET, request.getValue().getBucketName());
- Assert.assertEquals(PREFIX + "/" + TEST_FILE, request.getValue().getKey());
- EasyMock.verify(s3Client);
+ s3Client.putObject(
+ BUCKET,
+ JOINER.join(PREFIX, TEST_FILE),
+ Files.createFile(Path.of(temporaryFolder.toPath().toString(),
TEST_FILE)).toFile()
+ );
+ Assertions.assertTrue(storageConnector.pathExists(TEST_FILE));
}
@Test
public void pathExists_notFound() throws IOException
{
- final Capture<GetObjectMetadataRequest> request = Capture.newInstance();
- final AmazonS3Exception e = new AmazonS3Exception("not found");
- e.setStatusCode(404);
-
- EasyMock.reset(s3Client);
- EasyMock.expect(s3Client.getObjectMetadata(EasyMock.capture(request)))
- .andThrow(e);
- EasyMock.replay(s3Client);
- Assert.assertFalse(storageConnector.pathExists(TEST_FILE));
- Assert.assertEquals(BUCKET, request.getValue().getBucketName());
- Assert.assertEquals(PREFIX + "/" + TEST_FILE, request.getValue().getKey());
- EasyMock.verify(s3Client);
+
Assertions.assertFalse(storageConnector.pathExists(UUID.randomUUID().toString()));
}
@Test
- public void pathExists_error()
+ public void pathExists_error() throws IOException
{
- final Capture<GetObjectMetadataRequest> request = Capture.newInstance();
- final AmazonS3Exception e = new AmazonS3Exception("not found");
- e.setStatusCode(403);
-
- EasyMock.reset(s3Client);
- EasyMock.expect(s3Client.getObjectMetadata(EasyMock.capture(request)))
- .andThrow(e);
- EasyMock.replay(s3Client);
- final IOException e2 = Assert.assertThrows(
+ S3OutputConfig s3OutputConfig = new S3OutputConfig(
+ BUCKET,
+ PREFIX,
+ Files.createDirectory(Paths.get(temporaryFolder.getAbsolutePath(),
UUID.randomUUID().toString())).toFile(),
+ null,
+ null,
+ true
+ );
+ StorageConnector unauthorizedStorageConnector = new S3StorageConnector(
+ s3OutputConfig,
+ MinioUtil.createUnauthorizedS3Client(MINIO),
+ new S3UploadManager(
+ s3OutputConfig,
+ new S3ExportConfig("tempDir", new HumanReadableBytes("5MiB"), 1,
null),
+ new DruidProcessingConfigTest.MockRuntimeInfo(10, 0, 0),
+ new StubServiceEmitter()
+ )
+ );
+ final IOException e2 = Assertions.assertThrows(
IOException.class,
- () -> storageConnector.pathExists(TEST_FILE)
+ () -> unauthorizedStorageConnector.pathExists(TEST_FILE)
);
- Assert.assertEquals(BUCKET, request.getValue().getBucketName());
- Assert.assertEquals(PREFIX + "/" + TEST_FILE, request.getValue().getKey());
- MatcherAssert.assertThat(e2,
ThrowableCauseMatcher.hasCause(CoreMatchers.instanceOf(AmazonS3Exception.class)));
- EasyMock.verify(s3Client);
+ Assertions.assertEquals(AmazonS3Exception.class, e2.getCause().getClass());
+ AmazonS3Exception amazonS3Exception = (AmazonS3Exception) e2.getCause();
+ Assertions.assertEquals(403, amazonS3Exception.getStatusCode());
}
@Test
public void pathRead() throws IOException
{
- EasyMock.reset(s3Client);
- ObjectMetadata objectMetadata = new ObjectMetadata();
- long contentLength = "test".getBytes(StandardCharsets.UTF_8).length;
- objectMetadata.setContentLength(contentLength);
- S3Object s3Object = new S3Object();
- s3Object.setObjectContent(new
ByteArrayInputStream("test".getBytes(StandardCharsets.UTF_8)));
-
EasyMock.expect(s3Client.getObjectMetadata(EasyMock.anyObject())).andReturn(objectMetadata);
- EasyMock.expect(s3Client.getObject(
- new GetObjectRequest(BUCKET, PREFIX + "/" + TEST_FILE).withRange(0,
contentLength - 1))
- ).andReturn(s3Object);
- EasyMock.replay(s3Client);
-
- String readText = new BufferedReader(
- new InputStreamReader(storageConnector.read(TEST_FILE),
StandardCharsets.UTF_8))
- .lines()
- .collect(Collectors.joining("\n"));
-
- Assert.assertEquals("test", readText);
- EasyMock.reset(s3Client);
+ try (OutputStream outputStream = storageConnector.write("readWrite1")) {
+ outputStream.write("test".getBytes(StandardCharsets.UTF_8));
+ }
+ try (InputStream inputStream = storageConnector.read("readWrite1")) {
+ byte[] bytes = inputStream.readAllBytes();
+ Assertions.assertEquals("test", new String(bytes,
StandardCharsets.UTF_8));
+ }
}
@Test
@@ -180,139 +167,175 @@ public class S3StorageConnectorTest
{
String data = "test";
+ try (OutputStream outputStream = storageConnector.write("readWrite2")) {
+ outputStream.write(data.getBytes(StandardCharsets.UTF_8));
+ }
+
// non empty reads
for (int start = 0; start < data.length(); start++) {
for (int length = 1; length <= data.length() - start; length++) {
String dataQueried = data.substring(start, start + length);
- EasyMock.reset(s3Client);
- S3Object s3Object = new S3Object();
- s3Object.setObjectContent(
- new
ByteArrayInputStream(dataQueried.getBytes(StandardCharsets.UTF_8))
- );
- EasyMock.expect(
- s3Client.getObject(
- new GetObjectRequest(BUCKET, PREFIX + "/" +
TEST_FILE).withRange(start, start + length - 1)
- )
- ).andReturn(s3Object);
- EasyMock.replay(s3Client);
-
- InputStream is = storageConnector.readRange(TEST_FILE, start, length);
- byte[] dataBytes = new byte[length];
- Assert.assertEquals(length, is.read(dataBytes));
- Assert.assertEquals(-1, is.read()); // reading further produces no data
- Assert.assertEquals(dataQueried, new String(dataBytes,
StandardCharsets.UTF_8));
- EasyMock.reset(s3Client);
+ try (InputStream inputStream =
storageConnector.readRange("readWrite2", start, length)) {
+ byte[] bytes = inputStream.readAllBytes();
+ Assertions.assertEquals(dataQueried, new String(bytes,
StandardCharsets.UTF_8));
+ }
}
}
// empty read
- EasyMock.reset(s3Client);
- S3Object s3Object = new S3Object();
- s3Object.setObjectContent(
- new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8))
- );
- EasyMock.expect(
- s3Client.getObject(
- new GetObjectRequest(BUCKET, PREFIX + "/" +
TEST_FILE).withRange(0, -1)
- )
- ).andReturn(s3Object);
- EasyMock.replay(s3Client);
-
- InputStream is = storageConnector.readRange(TEST_FILE, 0, 0);
- byte[] dataBytes = new byte[0];
- Assert.assertEquals(is.read(dataBytes), -1);
- Assert.assertEquals("", new String(dataBytes, StandardCharsets.UTF_8));
- EasyMock.reset(s3Client);
+ try (InputStream inputStream = storageConnector.readRange("readWrite2", 0,
0)) {
+ byte[] bytes = inputStream.readAllBytes();
+ Assertions.assertEquals("", new String(bytes, StandardCharsets.UTF_8));
+ }
}
@Test
public void testDeleteSinglePath() throws IOException
{
- EasyMock.reset(s3Client);
- s3Client.deleteObject(BUCKET, PREFIX + "/" + TEST_FILE);
- EasyMock.expectLastCall();
- storageConnector.deleteFile(TEST_FILE);
- EasyMock.reset(s3Client);
+ String deleteFolderName = UUID.randomUUID().toString();
+ try (OutputStream outputStream =
storageConnector.write(StringUtils.format("%s/deleteSingle",
deleteFolderName))) {
+ outputStream.write("delete".getBytes(StandardCharsets.UTF_8));
+ }
+
+ ArrayList<String> listResult = new ArrayList<>();
+ storageConnector.listDir(deleteFolderName +
"/").forEachRemaining(listResult::add);
+ Assertions.assertEquals(1, listResult.size());
+ Assertions.assertEquals("deleteSingle", listResult.get(0));
+ storageConnector.deleteFile(StringUtils.format("%s/deleteSingle",
deleteFolderName));
+
+ listResult.clear();
+ storageConnector.listDir(deleteFolderName +
"/").forEachRemaining(listResult::add);
+ Assertions.assertEquals(0, listResult.size());
}
@Test
public void testDeleteMultiplePaths() throws IOException
{
- EasyMock.reset(s3Client);
- String testFile2 = "file2";
- DeleteObjectsRequest deleteObjectsRequest = new
DeleteObjectsRequest(BUCKET);
- deleteObjectsRequest.withKeys(PREFIX + "/" + TEST_FILE, PREFIX + "/" +
testFile2);
- Capture<DeleteObjectsRequest> capturedArgument = EasyMock.newCapture();
-
-
EasyMock.expect(s3Client.deleteObjects(EasyMock.capture(capturedArgument))).andReturn(null).once();
- EasyMock.replay(s3Client);
- storageConnector.deleteFiles(Lists.newArrayList(TEST_FILE, testFile2));
-
- Assert.assertEquals(
- convertDeleteObjectsRequestToString(deleteObjectsRequest),
- convertDeleteObjectsRequestToString(capturedArgument.getValue())
- );
- EasyMock.reset(s3Client);
+ String deleteFolderName = UUID.randomUUID().toString();
+ try (OutputStream outputStream =
storageConnector.write(StringUtils.format("%s/deleteFirst", deleteFolderName)))
{
+ outputStream.write("first".getBytes(StandardCharsets.UTF_8));
+ }
+ try (OutputStream outputStream =
storageConnector.write(StringUtils.format("%s/deleteSecond",
deleteFolderName))) {
+ outputStream.write("second".getBytes(StandardCharsets.UTF_8));
+ }
+
+ ArrayList<String> listResult = new ArrayList<>();
+ storageConnector.listDir(deleteFolderName +
"/").forEachRemaining(listResult::add);
+ Assertions.assertEquals(2, listResult.size());
+ Assertions.assertEquals("deleteFirst", listResult.get(0));
+ Assertions.assertEquals("deleteSecond", listResult.get(1));
+
+ storageConnector.deleteFiles(ImmutableList.of(
+ StringUtils.format("%s/deleteFirst", deleteFolderName),
+ StringUtils.format("%s/deleteSecond", deleteFolderName)
+ ));
+
+ listResult.clear();
+ storageConnector.listDir(deleteFolderName +
"/").forEachRemaining(listResult::add);
+ Assertions.assertEquals(0, listResult.size());
}
@Test
public void testPathDeleteRecursively() throws IOException
{
- EasyMock.reset(s3Client, testResult);
-
- S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
- s3ObjectSummary.setBucketName(BUCKET);
- s3ObjectSummary.setKey(PREFIX + "/test/" + TEST_FILE);
- s3ObjectSummary.setSize(1);
- EasyMock.expect(s3Client.listObjectsV2((ListObjectsV2Request)
EasyMock.anyObject()))
- .andReturn(testResult);
-
- EasyMock.expect(testResult.getBucketName()).andReturn("123").anyTimes();
-
EasyMock.expect(testResult.getObjectSummaries()).andReturn(Collections.singletonList(s3ObjectSummary)).anyTimes();
- EasyMock.expect(testResult.isTruncated()).andReturn(false).times(1);
- EasyMock.expect(testResult.getNextContinuationToken()).andReturn(null);
-
- Capture<DeleteObjectsRequest> capturedArgument = EasyMock.newCapture();
- EasyMock.expect(s3Client.deleteObjects(EasyMock.and(
- EasyMock.capture(capturedArgument),
- EasyMock.isA(DeleteObjectsRequest.class)
- ))).andReturn(null);
- EasyMock.replay(s3Client, testResult);
-
- storageConnector.deleteRecursively("test");
-
- Assert.assertEquals(1, capturedArgument.getValue().getKeys().size());
- Assert.assertEquals(PREFIX + "/test/" + TEST_FILE,
capturedArgument.getValue().getKeys().get(0).getKey());
- EasyMock.reset(s3Client, testResult);
+ String deleteFolderName = UUID.randomUUID().toString();
+ try (OutputStream outputStream =
storageConnector.write(StringUtils.format("%s/deleteFirst", deleteFolderName)))
{
+ outputStream.write("first".getBytes(StandardCharsets.UTF_8));
+ }
+ try (OutputStream outputStream =
storageConnector.write(StringUtils.format("%s/inner/deleteSecond",
deleteFolderName))) {
+ outputStream.write("second".getBytes(StandardCharsets.UTF_8));
+ }
+
+ ArrayList<String> listResult = new ArrayList<>();
+ storageConnector.listDir(deleteFolderName +
"/").forEachRemaining(listResult::add);
+ Assertions.assertEquals(2, listResult.size());
+ Assertions.assertEquals("deleteFirst", listResult.get(0));
+ Assertions.assertEquals("inner/deleteSecond", listResult.get(1));
+
+ storageConnector.deleteRecursively(deleteFolderName);
+
+ listResult.clear();
+ storageConnector.listDir(deleteFolderName +
"/").forEachRemaining(listResult::add);
+ Assertions.assertEquals(0, listResult.size());
}
@Test
public void testListDir() throws IOException
{
- EasyMock.reset(s3Client, testResult);
-
- S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
- s3ObjectSummary.setBucketName(BUCKET);
- s3ObjectSummary.setKey(PREFIX + "/test/" + TEST_FILE);
- s3ObjectSummary.setSize(1);
-
-
EasyMock.expect(testResult.getObjectSummaries()).andReturn(Collections.singletonList(s3ObjectSummary)).times(2);
- EasyMock.expect(testResult.isTruncated()).andReturn(false);
- EasyMock.expect(testResult.getNextContinuationToken()).andReturn(null);
- EasyMock.expect(s3Client.listObjectsV2((ListObjectsV2Request)
EasyMock.anyObject()))
- .andReturn(testResult);
- EasyMock.replay(s3Client, testResult);
-
- List<String> listDirResult =
Lists.newArrayList(storageConnector.listDir("test/"));
- Assert.assertEquals(ImmutableList.of(TEST_FILE), listDirResult);
+ String listFolderName = UUID.randomUUID().toString();
+ try (OutputStream outputStream =
storageConnector.write(StringUtils.format("%s/listFirst", listFolderName))) {
+ outputStream.write("first".getBytes(StandardCharsets.UTF_8));
+ }
+ List<String> listDirResult =
Lists.newArrayList(storageConnector.listDir(listFolderName + "/"));
+ Assertions.assertEquals(ImmutableList.of("listFirst"), listDirResult);
}
- private String convertDeleteObjectsRequestToString(DeleteObjectsRequest
deleteObjectsRequest)
+ // adapted from apache iceberg tests
+ private static class MinioUtil
{
- return deleteObjectsRequest.getKeys()
- .stream()
- .map(keyVersion -> keyVersion.getKey() +
keyVersion.getVersion())
- .collect(
- Collectors.joining());
+ private MinioUtil()
+ {
+ }
+
+ public static MinIOContainer createContainer()
+ {
+ return createContainer(null);
+ }
+
+ public static MinIOContainer createContainer(AWSCredentials credentials)
+ {
+ MinIOContainer container = new
MinIOContainer(DockerImageName.parse("minio/minio:latest"));
+
+ // this enables virtual-host-style requests. see
+ // https://github.com/minio/minio/tree/master/docs/config#domain
+ container.withEnv("MINIO_DOMAIN", "localhost");
+
+ if (credentials != null) {
+ container.withUserName(credentials.getAWSAccessKeyId());
+ container.withPassword(credentials.getAWSSecretKey());
+ }
+
+ return container;
+ }
+
+ public static ServerSideEncryptingAmazonS3 createS3Client(MinIOContainer
container)
+ {
+ final AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3Client
+ .builder()
+ .withEndpointConfiguration(
+ new AwsClientBuilder.EndpointConfiguration(
+ container.getS3URL(),
+ "us-east-1"
+ )
+ )
+ .withCredentials(new StaticCredentialsProvider(
+ new BasicAWSCredentials(container.getUserName(),
container.getPassword())))
+ .withClientConfiguration(new
ClientConfigurationFactory().getConfig())
+ .withPathStyleAccessEnabled(true); // OSX won't resolve subdomains
+
+ return ServerSideEncryptingAmazonS3.builder()
+
.setAmazonS3ClientBuilder(amazonS3ClientBuilder)
+ .build();
+ }
+
+ public static ServerSideEncryptingAmazonS3
createUnauthorizedS3Client(MinIOContainer container)
+ {
+ final AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3Client
+ .builder()
+ .withEndpointConfiguration(
+ new AwsClientBuilder.EndpointConfiguration(
+ container.getS3URL(),
+ "us-east-1"
+ )
+ )
+ .withCredentials(new StaticCredentialsProvider(
+ new BasicAWSCredentials(container.getUserName(), "wrong")))
+ .withClientConfiguration(new
ClientConfigurationFactory().getConfig())
+ .withPathStyleAccessEnabled(true); // OSX won't resolve subdomains
+
+ return ServerSideEncryptingAmazonS3.builder()
+
.setAmazonS3ClientBuilder(amazonS3ClientBuilder)
+ .build();
+ }
}
}
diff --git a/pom.xml b/pom.xml
index ac9b91d23d0..3ee7600d543 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1403,6 +1403,18 @@
<artifactId>joni</artifactId>
<version>2.1.34</version>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <version>1.16.3</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>junit-jupiter</artifactId>
+ <version>1.16.3</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -2257,5 +2269,8 @@
<jacoco.skip>true</jacoco.skip>
</properties>
</profile>
+ <profile>
+ <id>ci</id>
+ </profile>
</profiles>
</project>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
