This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 6b856962d9c Embedded tests using MinIO deep storage. (#18202)
6b856962d9c is described below
commit 6b856962d9c052a9c656c6a243203e6758cc7190
Author: Gian Merlino <[email protected]>
AuthorDate: Sun Jul 6 14:15:50 2025 -0700
Embedded tests using MinIO deep storage. (#18202)
* Embedded tests using MinIO deep storage.
Adds a testcontainers-based MinIOStorageResource that configures the
cluster to use MinIO as S3-compatible deep storage.
* Add missing dependencies.
---
embedded-tests/pom.xml | 27 +++++
.../embedded/minio/EmbeddedMinIOStorageTest.java | 44 +++++++
.../embedded/minio/MinIOStorageResource.java | 134 +++++++++++++++++++++
.../embedded/minio/MinIOStorageResourceTest.java | 74 ++++++++++++
4 files changed, 279 insertions(+)
diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml
index 6458dc01dc4..15025a557bb 100644
--- a/embedded-tests/pom.xml
+++ b/embedded-tests/pom.xml
@@ -66,6 +66,12 @@
<version>${project.parent.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.druid.extensions</groupId>
+ <artifactId>druid-s3-extensions</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
@@ -188,6 +194,27 @@
<version>${mariadb.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>minio</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-core</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/EmbeddedMinIOStorageTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/EmbeddedMinIOStorageTest.java
new file mode 100644
index 00000000000..4bc5aa4737a
--- /dev/null
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/EmbeddedMinIOStorageTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.minio;
+
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.EmbeddedRouter;
+import org.apache.druid.testing.embedded.indexing.EmbeddedIndexTaskTest;
+
+/**
+ * Same as {@link EmbeddedIndexTaskTest}, but using a MinIO metadata store
through the S3 extension.
+ */
+public class EmbeddedMinIOStorageTest extends EmbeddedIndexTaskTest
+{
+ @Override
+ public EmbeddedDruidCluster createCluster()
+ {
+ return EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper()
+ .useLatchableEmitter()
+ .addResource(new MinIOStorageResource())
+ .addServer(coordinator)
+ .addServer(indexer)
+ .addServer(overlord)
+ .addServer(historical)
+ .addServer(broker)
+ .addServer(new EmbeddedRouter());
+ }
+}
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResource.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResource.java
new file mode 100644
index 00000000000..4a24d370f35
--- /dev/null
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResource.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.minio;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import org.apache.druid.storage.s3.S3StorageDruidModule;
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.TestcontainerResource;
+import org.testcontainers.containers.MinIOContainer;
+
+/**
+ * A MinIO container resource for use in embedded tests as deep storage.
+ * Sets up MinIO as S3-compatible storage and configures Druid's S3 connector.
+ */
+public class MinIOStorageResource extends TestcontainerResource<MinIOContainer>
+{
+ private static final String MINIO_IMAGE = "minio/minio:latest";
+ private static final String DEFAULT_BUCKET = "druid-deep-storage";
+ private static final String DEFAULT_BASE_KEY = "druid/segments";
+ private static final String ACCESS_KEY = "minioadmin";
+ private static final String SECRET_KEY = "minioadmin";
+
+ private final String bucket;
+ private final String baseKey;
+ private AmazonS3 s3Client;
+
+ public MinIOStorageResource()
+ {
+ this(DEFAULT_BUCKET, DEFAULT_BASE_KEY);
+ }
+
+ public MinIOStorageResource(String bucket, String baseKey)
+ {
+ this.bucket = bucket;
+ this.baseKey = baseKey;
+ }
+
+ @Override
+ protected MinIOContainer createContainer()
+ {
+ return new MinIOContainer(MINIO_IMAGE)
+ .withUserName(getAccessKey())
+ .withPassword(getSecretKey());
+ }
+
+ @Override
+ public void onStarted(EmbeddedDruidCluster cluster)
+ {
+ s3Client = createS3Client();
+ s3Client.createBucket(bucket);
+
+ cluster.addExtension(S3StorageDruidModule.class);
+
+ // Configure storage bucket and base key
+ cluster.addCommonProperty("druid.storage.type", "s3");
+ cluster.addCommonProperty("druid.storage.bucket", getBucket());
+ cluster.addCommonProperty("druid.storage.baseKey", getBaseKey());
+
+ // Configure indexer logs
+ cluster.addCommonProperty("druid.indexer.logs.type", "s3");
+ cluster.addCommonProperty("druid.indexer.logs.s3Bucket", getBucket());
+ cluster.addCommonProperty("druid.indexer.logs.s3Prefix",
"druid/indexing-logs");
+
+ // Configure S3 connection properties
+ cluster.addCommonProperty("druid.s3.endpoint.url", getEndpointUrl());
+ cluster.addCommonProperty("druid.s3.accessKey", getAccessKey());
+ cluster.addCommonProperty("druid.s3.secretKey", getSecretKey());
+ cluster.addCommonProperty("druid.s3.enablePathStyleAccess", "true");
+ cluster.addCommonProperty("druid.s3.protocol", "http");
+ }
+
+ public String getBucket()
+ {
+ return bucket;
+ }
+
+ public String getBaseKey()
+ {
+ return baseKey;
+ }
+
+ public String getAccessKey()
+ {
+ return ACCESS_KEY;
+ }
+
+ public String getSecretKey()
+ {
+ return SECRET_KEY;
+ }
+
+ public String getEndpointUrl()
+ {
+ ensureRunning();
+ return getContainer().getS3URL();
+ }
+
+ public AmazonS3 getS3Client()
+ {
+ ensureRunning();
+ return s3Client;
+ }
+
+ private AmazonS3 createS3Client()
+ {
+ return AmazonS3Client
+ .builder()
+ .withEndpointConfiguration(new
AwsClientBuilder.EndpointConfiguration(getEndpointUrl(), "us-east-1"))
+ .withCredentials(new AWSStaticCredentialsProvider(new
BasicAWSCredentials(getAccessKey(), getSecretKey())))
+ .withPathStyleAccessEnabled(true)
+ .build();
+ }
+}
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResourceTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResourceTest.java
new file mode 100644
index 00000000000..a7569a96395
--- /dev/null
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResourceTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.minio;
+
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.verify;
+
+public class MinIOStorageResourceTest
+{
+ @Test
+ @Timeout(120)
+ public void testMinIOContainerLifecycle()
+ {
+ final MinIOStorageResource resource = new
MinIOStorageResource("test-bucket", "test/base");
+ final EmbeddedDruidCluster cluster =
Mockito.mock(EmbeddedDruidCluster.class);
+
+ resource.start();
+ assertTrue(resource.isRunning());
+
+ resource.onStarted(cluster);
+
+ // Verify container properties
+ assertEquals("test-bucket", resource.getBucket());
+ assertEquals("test/base", resource.getBaseKey());
+ assertEquals("minioadmin", resource.getAccessKey());
+ assertEquals("minioadmin", resource.getSecretKey());
+
+ // Verify all the required properties are set
+ verify(cluster).addCommonProperty("druid.storage.type", "s3");
+ verify(cluster).addCommonProperty("druid.indexer.logs.type", "s3");
+ verify(cluster).addCommonProperty("druid.s3.enablePathStyleAccess",
"true");
+ verify(cluster).addCommonProperty("druid.s3.protocol", "http");
+ verify(cluster).addCommonProperty("druid.s3.accessKey", "minioadmin");
+ verify(cluster).addCommonProperty("druid.s3.secretKey", "minioadmin");
+ verify(cluster).addCommonProperty("druid.storage.bucket", "test-bucket");
+ verify(cluster).addCommonProperty("druid.storage.baseKey", "test/base");
+ verify(cluster).addCommonProperty("druid.indexer.logs.s3Bucket",
"test-bucket");
+ verify(cluster).addCommonProperty("druid.indexer.logs.s3Prefix",
"druid/indexing-logs");
+
+ // Verify endpoint URL is set
+ verify(cluster).addCommonProperty(
+ ArgumentMatchers.eq("druid.s3.endpoint.url"),
+ ArgumentMatchers.argThat(url -> url.startsWith("http://";))
+ );
+
+ resource.stop();
+ assertFalse(resource.isRunning());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
