This is an automated email from the ASF dual-hosted git repository.
FrankChen021 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new a01e318f96c minor: use rustfs instead of minio for s3 embedded-tests
(#20420)
a01e318f96c is described below
commit a01e318f96c97912c227cfba2f946bc5cab37a03
Author: Clint Wylie <[email protected]>
AuthorDate: Thu Sep 24 18:59:19 2026 -0700
minor: use rustfs instead of minio for s3 embedded-tests (#20420)
---
.github/workflows/worker.yml | 2 +-
embedded-tests/pom.xml | 6 --
.../docker/BaseConsulDiscoveryDockerTest.java | 10 +--
.../embedded/hdfs/HdfsToS3ParallelIndexTest.java | 14 ++--
.../embedded/hdfs/S3ToHdfsParallelIndexTest.java | 10 +--
.../AbstractS3InputSourceParallelIndexTest.java | 16 ++---
.../embedded/indexing/IngestionSmokeTest.java | 4 +-
.../msq/EmbeddedDurableShuffleStorageTest.java | 8 +--
...Resource.java => S3DurableStorageResource.java} | 12 ++--
.../testing/embedded/msq/S3ExternQueryTest.java | 8 +--
.../query/PartialProjectionLoadRuleQueryTest.java | 6 +-
.../embedded/query/QueryVirtualStorageTest.java | 4 +-
.../{minio => s3}/ITS3SQLBasedIngestionTest.java | 2 +-
.../{minio => s3}/ITS3ToS3ParallelIndexTest.java | 6 +-
.../testing/embedded/{minio => s3}/S3PerfTest.java | 20 +++---
.../S3StorageResource.java} | 44 +++++++------
.../S3StorageResourceTest.java} | 16 ++---
.../S3StorageTest.java} | 8 +--
.../testing/embedded/{minio => s3}/S3TestUtil.java | 2 +-
extensions-core/s3-extensions/pom.xml | 6 --
.../apache/druid/storage/s3/RustFSContainer.java | 77 ++++++++++++++++++++++
...rSideEncryptingAmazonS3MultipartUploadTest.java | 13 ++--
.../storage/s3/output/S3StorageConnectorTest.java | 55 ++++------------
23 files changed, 191 insertions(+), 158 deletions(-)
diff --git a/.github/workflows/worker.yml b/.github/workflows/worker.yml
index 6c7de6aae22..1b08a3ceb75 100644
--- a/.github/workflows/worker.yml
+++ b/.github/workflows/worker.yml
@@ -93,7 +93,7 @@ jobs:
images=(
apache/kafka:4.2.0
postgres:16-alpine
- quay.io/minio/minio:latest
+ rustfs/rustfs:1.0.0
hashicorp/consul:1.18
tabulario/iceberg-rest:1.6.0
localstack/localstack:4.13.1
diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml
index 140c7959724..d9052fea9c1 100644
--- a/embedded-tests/pom.xml
+++ b/embedded-tests/pom.xml
@@ -587,12 +587,6 @@
<version>${mariadb.version}</version>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>testcontainers-minio</artifactId>
- <version>${testcontainers.version}</version>
- <scope>test</scope>
- </dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-redpanda</artifactId>
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/docker/BaseConsulDiscoveryDockerTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/docker/BaseConsulDiscoveryDockerTest.java
index 216630a1140..a3002b331ec 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/docker/BaseConsulDiscoveryDockerTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/docker/BaseConsulDiscoveryDockerTest.java
@@ -33,8 +33,8 @@ import
org.apache.druid.testing.embedded.consul.ConsulClusterResource;
import org.apache.druid.testing.embedded.consul.ConsulSecurityMode;
import org.apache.druid.testing.embedded.emitter.LatchableEmitterModule;
import org.apache.druid.testing.embedded.indexing.IngestionSmokeTest;
-import org.apache.druid.testing.embedded.minio.MinIOStorageResource;
import org.apache.druid.testing.embedded.psql.PostgreSQLMetadataResource;
+import org.apache.druid.testing.embedded.s3.S3StorageResource;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
@@ -66,7 +66,7 @@ abstract class BaseConsulDiscoveryDockerTest extends
IngestionSmokeTest
public EmbeddedDruidCluster createCluster()
{
final PostgreSQLMetadataResource postgreSQLMetadataResource = new
PostgreSQLMetadataResource();
- final MinIOStorageResource minIOStorageResource = new
MinIOStorageResource();
+ final S3StorageResource s3StorageResource = new S3StorageResource();
consulResource = new ConsulClusterResource(getConsulSecurityMode());
@@ -81,7 +81,7 @@ abstract class BaseConsulDiscoveryDockerTest extends
IngestionSmokeTest
);
cluster.addResource(postgreSQLMetadataResource)
- .addResource(minIOStorageResource)
+ .addResource(s3StorageResource)
.addResource(consulResource)
.addResource(kafkaServer);
@@ -125,12 +125,12 @@ abstract class BaseConsulDiscoveryDockerTest extends
IngestionSmokeTest
String consulHost = "localhost";
String consulPort = Integer.toString(consulResource.getMappedPort());
- String minioUrl = minIOStorageResource.getEndpointUrl();
+ String s3Url = s3StorageResource.getEndpointUrl();
cluster.addCommonProperty("druid.metadata.storage.connector.connectURI",
psqlConnectURI);
cluster.addCommonProperty("druid.discovery.consul.connection.host",
consulHost);
cluster.addCommonProperty("druid.discovery.consul.connection.port",
consulPort);
- cluster.addCommonProperty("druid.s3.endpoint.url", minioUrl);
+ cluster.addCommonProperty("druid.s3.endpoint.url", s3Url);
ConsulSecurityMode mode = getConsulSecurityMode();
if (mode == ConsulSecurityMode.TLS || mode == ConsulSecurityMode.MTLS)
{
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/hdfs/HdfsToS3ParallelIndexTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/hdfs/HdfsToS3ParallelIndexTest.java
index 2c57f70dc46..62ce85450a0 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/hdfs/HdfsToS3ParallelIndexTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/hdfs/HdfsToS3ParallelIndexTest.java
@@ -22,15 +22,15 @@ package org.apache.druid.testing.embedded.hdfs;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
-import org.apache.druid.testing.embedded.minio.MinIOStorageResource;
+import org.apache.druid.testing.embedded.s3.S3StorageResource;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
/**
* Embedded parallel index test that reads input data from HDFS (in-process
MiniDFSCluster)
- * and stores segments in S3 (MinIO testcontainer).
+ * and stores segments in S3 (an S3-compatible testcontainer).
*
- * <p>The MinIO resource is registered after the HDFS resource so that the S3
deep-storage
+ * <p>The S3 resource is registered after the HDFS resource so that the S3
deep-storage
* configuration ({@code druid.storage.type=s3}) is applied last. The HDFS
input-source
* connection properties ({@code hadoop.fs.defaultFS}) remain active and are
used by the
* indexer to read data from HDFS.
@@ -39,9 +39,9 @@ public class HdfsToS3ParallelIndexTest extends
AbstractHdfsInputSourceParallelIn
{
private static final Logger LOG = new
Logger(HdfsToS3ParallelIndexTest.class);
- /** HDFS is the input source only — deep storage is S3/MinIO. */
+ /** HDFS is the input source only — deep storage is S3. */
private final HdfsStorageResource hdfsResource = new
HdfsStorageResource(false);
- private final MinIOStorageResource minIOResource = new
MinIOStorageResource();
+ private final S3StorageResource s3Resource = new S3StorageResource();
@Override
protected HdfsStorageResource getHdfsResource()
@@ -56,9 +56,9 @@ public class HdfsToS3ParallelIndexTest extends
AbstractHdfsInputSourceParallelIn
// can read from HDFS. configureAsDeepStorage=false means it does NOT set
druid.storage.type.
cluster.addResource(hdfsResource);
- // MinIO resource: configures S3/MinIO as deep storage
(druid.storage.type=s3, etc.).
+ // S3 resource: configures S3 as deep storage (druid.storage.type=s3,
etc.).
// Adding it after the HDFS resource ensures the S3 deep-storage settings
win.
- cluster.addResource(minIOResource);
+ cluster.addResource(s3Resource);
}
@ParameterizedTest
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/hdfs/S3ToHdfsParallelIndexTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/hdfs/S3ToHdfsParallelIndexTest.java
index 23063af355d..e44d0bc6cd6 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/hdfs/S3ToHdfsParallelIndexTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/hdfs/S3ToHdfsParallelIndexTest.java
@@ -28,13 +28,13 @@ import org.junit.jupiter.params.provider.MethodSource;
import java.util.List;
/**
- * Embedded parallel index test that reads input data from S3 (MinIO
testcontainer) and stores
+ * Embedded parallel index test that reads input data from S3 (an
S3-compatible testcontainer) and stores
* segments in HDFS (in-process MiniDFSCluster).
*
- * <p>The HDFS resource is registered after the MinIO resource so that the
HDFS deep-storage
- * configuration ({@code druid.storage.type=hdfs}) overrides the S3
configuration set by MinIO.
+ * <p>The HDFS resource is registered after the S3 resource so that the HDFS
deep-storage
+ * configuration ({@code druid.storage.type=hdfs}) overrides the S3
configuration set by the S3 resource.
* The S3 input-source connection properties remain active and are used by the
indexer to read
- * data from MinIO.
+ * data from S3.
*/
public class S3ToHdfsParallelIndexTest extends
AbstractS3InputSourceParallelIndexTest
{
@@ -43,7 +43,7 @@ public class S3ToHdfsParallelIndexTest extends
AbstractS3InputSourceParallelInde
@Override
protected void addResources(EmbeddedDruidCluster cluster)
{
- // S3/MinIO resource first: uploads data to MinIO and sets S3 connection
properties.
+ // S3 resource first: uploads data to S3 and sets S3 connection properties.
super.addResources(cluster);
// HDFS resource second: overrides druid.storage.type to "hdfs" for deep
storage.
cluster.addResource(hdfsResource);
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractS3InputSourceParallelIndexTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractS3InputSourceParallelIndexTest.java
index 997a884a821..6029495dbfb 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractS3InputSourceParallelIndexTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/AbstractS3InputSourceParallelIndexTest.java
@@ -26,8 +26,8 @@ import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
-import org.apache.druid.testing.embedded.minio.MinIOStorageResource;
-import org.apache.druid.testing.embedded.minio.S3TestUtil;
+import org.apache.druid.testing.embedded.s3.S3StorageResource;
+import org.apache.druid.testing.embedded.s3.S3TestUtil;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
@@ -50,14 +50,14 @@ public abstract class
AbstractS3InputSourceParallelIndexTest extends AbstractClo
private static final Logger LOG = new
Logger(AbstractS3InputSourceParallelIndexTest.class);
private static final String INDEX_TASK =
"/indexer/wikipedia_cloud_index_task.json";
private static final String INDEX_QUERIES_RESOURCE =
"/indexer/wikipedia_index_queries.json";
- protected final MinIOStorageResource minIOStorageResource = new
MinIOStorageResource();
+ protected final S3StorageResource s3StorageResource = new
S3StorageResource();
private S3TestUtil s3;
@Override
protected void addResources(EmbeddedDruidCluster cluster)
{
cluster.addExtension(S3InputSourceDruidModule.class)
- .addResource(minIOStorageResource);
+ .addResource(s3StorageResource);
}
@BeforeAll
@@ -70,7 +70,7 @@ public abstract class AbstractS3InputSourceParallelIndexTest
extends AbstractClo
}
try {
s3 = new S3TestUtil(
- minIOStorageResource.getS3Client(),
+ s3StorageResource.getS3Client(),
getCloudBucket("s3"),
getCloudPath("s3")
);
@@ -101,7 +101,7 @@ public abstract class
AbstractS3InputSourceParallelIndexTest extends AbstractClo
/**
* Variant of {@link #doTest} that injects an {@code endpointConfig} into
the S3 input source spec.
* Required when using per-input-source credentials: S3InputSource builds a
new S3 client in that
- * case and must be told the MinIO endpoint explicitly, since it cannot
inherit it from Druid's
+ * case and must be told the S3 endpoint explicitly, since it cannot inherit
it from Druid's
* global S3 config.
*/
protected void doTestWithEndpointConfig(
@@ -116,7 +116,7 @@ public abstract class
AbstractS3InputSourceParallelIndexTest extends AbstractClo
final String endpointConfigJson = jsonMapper.writeValueAsString(
Map.of("url", endpointUrl, "signingRegion", "us-east-1")
);
- // Path-style access is required for MinIO running at a local IP address
+ // Path-style access is required for the S3 container running at a local
IP address
final String clientConfigJson = jsonMapper.writeValueAsString(
Map.of("enablePathStyleAccess", true)
);
@@ -140,7 +140,7 @@ public abstract class
AbstractS3InputSourceParallelIndexTest extends AbstractClo
);
spec = StringUtils.replace(spec, "%%INPUT_SOURCE_PROPERTY_KEY%%",
inputSource.lhs);
spec = StringUtils.replace(spec, "%%INPUT_SOURCE_PROPERTY_VALUE%%",
inputSourceValue);
- // Inject endpointConfig and clientConfig so the custom S3 client
points at MinIO with path-style access
+ // Inject endpointConfig and clientConfig so the custom S3 client
points at the S3 container with path-style access
spec = StringUtils.replace(
spec,
"\"type\": \"s3\",",
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IngestionSmokeTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IngestionSmokeTest.java
index d703325ce6e..2d22e80960b 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IngestionSmokeTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/IngestionSmokeTest.java
@@ -51,9 +51,9 @@ import org.apache.druid.testing.embedded.EmbeddedOverlord;
import org.apache.druid.testing.embedded.EmbeddedRouter;
import org.apache.druid.testing.embedded.emitter.LatchableEmitterModule;
import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
-import org.apache.druid.testing.embedded.minio.MinIOStorageResource;
import org.apache.druid.testing.embedded.msq.EmbeddedMSQApis;
import org.apache.druid.testing.embedded.psql.PostgreSQLMetadataResource;
+import org.apache.druid.testing.embedded.s3.S3StorageResource;
import org.apache.druid.testing.embedded.server.EmbeddedEventCollector;
import org.apache.druid.timeline.DataSegment;
import org.apache.kafka.clients.producer.ProducerRecord;
@@ -116,7 +116,7 @@ public class IngestionSmokeTest extends
EmbeddedClusterTestBase
)
.useDefaultTimeoutForLatchableEmitter(20)
.addResource(new PostgreSQLMetadataResource())
- .addResource(new MinIOStorageResource())
+ .addResource(new S3StorageResource())
.addResource(kafkaServer)
.addCommonProperty("druid.manager.segments.useIncrementalCache",
"always")
.addCommonProperty("druid.emitter", "http")
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java
index 05fd2d02094..9c2d3c277c8 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java
@@ -40,7 +40,7 @@ import org.apache.druid.testing.embedded.EmbeddedIndexer;
import org.apache.druid.testing.embedded.EmbeddedOverlord;
import org.apache.druid.testing.embedded.EmbeddedRouter;
import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
-import org.apache.druid.testing.embedded.minio.MinIOStorageResource;
+import org.apache.druid.testing.embedded.s3.S3StorageResource;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.junit.jupiter.api.Assertions;
@@ -60,7 +60,7 @@ import java.util.stream.Collectors;
/**
* Embedded test to batch-ingest wikipedia data from {@link
#loadWikipediaTable()}, then query that data
- * with MSQ tasks using MinIO-based durable storage.
+ * with MSQ tasks using S3-based durable storage.
*/
public class EmbeddedDurableShuffleStorageTest extends EmbeddedClusterTestBase
{
@@ -70,8 +70,8 @@ public class EmbeddedDurableShuffleStorageTest extends
EmbeddedClusterTestBase
private final EmbeddedHistorical historical = new EmbeddedHistorical();
private final EmbeddedCoordinator coordinator = new EmbeddedCoordinator();
private final EmbeddedRouter router = new EmbeddedRouter();
- private final MinIOStorageResource storageResource = new
MinIOStorageResource();
- private final MinIODurableStorageResource msqStorageResource = new
MinIODurableStorageResource(storageResource);
+ private final S3StorageResource storageResource = new S3StorageResource();
+ private final S3DurableStorageResource msqStorageResource = new
S3DurableStorageResource(storageResource);
private EmbeddedMSQApis msqApis;
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/MinIODurableStorageResource.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/S3DurableStorageResource.java
similarity index 85%
rename from
embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/MinIODurableStorageResource.java
rename to
embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/S3DurableStorageResource.java
index 8913e3e3ea4..509f8dffa84 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/MinIODurableStorageResource.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/S3DurableStorageResource.java
@@ -22,18 +22,18 @@ package org.apache.druid.testing.embedded.msq;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
import org.apache.druid.testing.embedded.EmbeddedResource;
-import org.apache.druid.testing.embedded.minio.MinIOStorageResource;
+import org.apache.druid.testing.embedded.s3.S3StorageResource;
import java.io.File;
/**
- * Resource that configures MSQ to use a bucket from {@link
MinIOStorageResource} for durable intermediate data storage.
+ * Resource that configures MSQ to use a bucket from {@link S3StorageResource}
for durable intermediate data storage.
*/
-public class MinIODurableStorageResource implements EmbeddedResource
+public class S3DurableStorageResource implements EmbeddedResource
{
- private final MinIOStorageResource storageResource;
+ private final S3StorageResource storageResource;
- public MinIODurableStorageResource(MinIOStorageResource storageResource)
+ public S3DurableStorageResource(S3StorageResource storageResource)
{
this.storageResource = storageResource;
}
@@ -63,7 +63,7 @@ public class MinIODurableStorageResource implements
EmbeddedResource
// Set tmpStorageBytesPerTask to 3 GB. This controls when stage-internal
channels (such as ones used internally
// by SuperSorter) "spill over" from local disk to durable storage. Cannot
set this much lower currently, due to
// validations in WorkerStorageParameters. Ideally, we'd like to set this
low enough such that embedded tests are
- // actually using minio/S3 for stage-internal storage. At this level, they
won't be using it, but at least stages
+ // actually using S3 for stage-internal storage. At this level, they won't
be using it, but at least stages
// will go through the motions of setting up composing channels
internally. There is still value in exercising
// that code path.
//
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/S3ExternQueryTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/S3ExternQueryTest.java
index 8eb06e76a39..56f43e0ed1f 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/S3ExternQueryTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/S3ExternQueryTest.java
@@ -29,7 +29,7 @@ import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
import org.apache.druid.testing.embedded.EmbeddedIndexer;
import org.apache.druid.testing.embedded.EmbeddedOverlord;
import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
-import org.apache.druid.testing.embedded.minio.MinIOStorageResource;
+import org.apache.druid.testing.embedded.s3.S3StorageResource;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
@@ -44,7 +44,7 @@ import java.util.List;
import java.util.zip.GZIPOutputStream;
/**
- * Tests MSQ task-based SELECT queries that read external data via {@code
EXTERN} from S3 (backed by MinIO).
+ * Tests MSQ task-based SELECT queries that read external data via {@code
EXTERN} from S3 (backed by an S3-compatible testcontainer).
*/
public class S3ExternQueryTest extends EmbeddedClusterTestBase
{
@@ -72,7 +72,7 @@ public class S3ExternQueryTest extends EmbeddedClusterTestBase
*/
private long totalUploadedBytes;
- private final MinIOStorageResource storageResource = new
MinIOStorageResource();
+ private final S3StorageResource storageResource = new S3StorageResource();
private final EmbeddedOverlord overlord = new EmbeddedOverlord();
private final EmbeddedCoordinator coordinator = new EmbeddedCoordinator();
private final EmbeddedIndexer indexer = new EmbeddedIndexer()
@@ -177,7 +177,7 @@ public class S3ExternQueryTest extends
EmbeddedClusterTestBase
}
/**
- * Uploads {@link #PLAIN_FILE} (plain JSON) and {@link #GZ_FILE} (gzipped
JSON) to the MinIO bucket.
+ * Uploads {@link #PLAIN_FILE} (plain JSON) and {@link #GZ_FILE} (gzipped
JSON) to the S3 bucket.
*/
private void uploadExternalFiles() throws IOException
{
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/PartialProjectionLoadRuleQueryTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/PartialProjectionLoadRuleQueryTest.java
index 925fb78722a..a4dd81e1144 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/PartialProjectionLoadRuleQueryTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/PartialProjectionLoadRuleQueryTest.java
@@ -57,8 +57,8 @@ import org.apache.druid.testing.embedded.EmbeddedOverlord;
import org.apache.druid.testing.embedded.EmbeddedRouter;
import org.apache.druid.testing.embedded.catalog.TestCatalogClient;
import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
-import org.apache.druid.testing.embedded.minio.MinIOStorageResource;
import org.apache.druid.testing.embedded.msq.EmbeddedMSQApis;
+import org.apache.druid.testing.embedded.s3.S3StorageResource;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.junit.jupiter.api.Assertions;
@@ -112,7 +112,7 @@ class PartialProjectionLoadRuleQueryTest extends
EmbeddedClusterTestBase
private final EmbeddedHistorical historical = new EmbeddedHistorical();
private final EmbeddedCoordinator coordinator = new EmbeddedCoordinator();
private final EmbeddedRouter router = new EmbeddedRouter();
- private final MinIOStorageResource storageResource = new
MinIOStorageResource();
+ private final S3StorageResource storageResource = new S3StorageResource();
private EmbeddedMSQApis msqApis;
@@ -176,7 +176,7 @@ class PartialProjectionLoadRuleQueryTest extends
EmbeddedClusterTestBase
.addCommonProperty("druid.msq.dart.enabled", "true")
// Clustered base-table + projection segments require the V10 segment
format.
.addCommonProperty("druid.indexer.task.buildV10", "true")
- // Range reads over MinIO — required for partial loading to be
functional.
+ // Range reads over S3 — required for partial loading to be functional.
.addCommonProperty("druid.storage.zip", "false")
.addCommonProperty("druid.monitoring.emissionPeriod", "PT1s")
.addServer(coordinator)
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
index 466f2cb9eb0..bf054bc3063 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/QueryVirtualStorageTest.java
@@ -43,9 +43,9 @@ import org.apache.druid.testing.embedded.EmbeddedIndexer;
import org.apache.druid.testing.embedded.EmbeddedOverlord;
import org.apache.druid.testing.embedded.EmbeddedRouter;
import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
-import org.apache.druid.testing.embedded.minio.MinIOStorageResource;
import org.apache.druid.testing.embedded.msq.EmbeddedDurableShuffleStorageTest;
import org.apache.druid.testing.embedded.msq.EmbeddedMSQApis;
+import org.apache.druid.testing.embedded.s3.S3StorageResource;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.junit.jupiter.api.Assertions;
@@ -83,7 +83,7 @@ class QueryVirtualStorageTest extends EmbeddedClusterTestBase
private final EmbeddedHistorical historical = new EmbeddedHistorical();
private final EmbeddedCoordinator coordinator = new EmbeddedCoordinator();
private final EmbeddedRouter router = new EmbeddedRouter();
- private final MinIOStorageResource storageResource = new
MinIOStorageResource();
+ private final S3StorageResource storageResource = new S3StorageResource();
private EmbeddedMSQApis msqApis;
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/ITS3SQLBasedIngestionTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/ITS3SQLBasedIngestionTest.java
similarity index 96%
rename from
embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/ITS3SQLBasedIngestionTest.java
rename to
embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/ITS3SQLBasedIngestionTest.java
index 57b3183226e..0a513fc33f7 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/ITS3SQLBasedIngestionTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/ITS3SQLBasedIngestionTest.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.druid.testing.embedded.minio;
+package org.apache.druid.testing.embedded.s3;
import org.apache.druid.java.util.common.Pair;
import
org.apache.druid.testing.embedded.indexer.AbstractS3InputSourceParallelIndexTest;
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/ITS3ToS3ParallelIndexTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/ITS3ToS3ParallelIndexTest.java
similarity index 90%
rename from
embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/ITS3ToS3ParallelIndexTest.java
rename to
embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/ITS3ToS3ParallelIndexTest.java
index cfd4a64960a..2e9e014585c 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/ITS3ToS3ParallelIndexTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/ITS3ToS3ParallelIndexTest.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.druid.testing.embedded.minio;
+package org.apache.druid.testing.embedded.s3;
import org.apache.druid.java.util.common.Pair;
import
org.apache.druid.testing.embedded.indexer.AbstractS3InputSourceParallelIndexTest;
@@ -42,8 +42,8 @@ public class ITS3ToS3ParallelIndexTest extends
AbstractS3InputSourceParallelInde
doTestWithEndpointConfig(
s3InputSource,
new Pair<>(false, false),
- minIOStorageResource.createTempCredentialsForInputSource(),
- minIOStorageResource.getEndpointUrl()
+ s3StorageResource.createTempCredentialsForInputSource(),
+ s3StorageResource.getEndpointUrl()
);
}
}
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/S3PerfTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3PerfTest.java
similarity index 94%
rename from
embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/S3PerfTest.java
rename to
embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3PerfTest.java
index 3446998f3f2..c63aa5e0606 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/S3PerfTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3PerfTest.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.druid.testing.embedded.minio;
+package org.apache.druid.testing.embedded.s3;
import org.apache.druid.common.aws.AWSModule;
import org.apache.druid.indexing.common.task.IndexTask;
@@ -62,7 +62,7 @@ import java.util.concurrent.TimeUnit;
* <p>Each {@link Nested} inner class runs a full embedded cluster with a
different
* storage backend / HTTP client combination:
* <ul>
- * <li>MinIO container — CRT (default), Netty NIO, synchronous PutObject</li>
+ * <li>Local S3-compatible container — CRT (default), Netty NIO, synchronous
PutObject</li>
* <li>Real AWS S3 (opt-in via {@value BUCKET_PROPERTY}) — same three
variants</li>
* </ul>
*
@@ -165,22 +165,22 @@ public class S3PerfTest
}
}
- abstract static class MinIOPerfBase extends PerfTestBase
+ abstract static class LocalS3PerfBase extends PerfTestBase
{
abstract void configureCluster(EmbeddedDruidCluster cluster);
@Override
public EmbeddedDruidCluster createCluster()
{
- final EmbeddedDruidCluster cluster =
addServers(newBaseCluster().addResource(new MinIOStorageResource()));
+ final EmbeddedDruidCluster cluster =
addServers(newBaseCluster().addResource(new S3StorageResource()));
configureCluster(cluster);
return cluster;
}
}
- /** MinIO + Amazon CRT async HTTP client (default). */
+ /** Local S3 container + Amazon CRT async HTTP client (default). */
@Nested
- static class MinIO_Crt extends MinIOPerfBase
+ static class LocalS3_Crt extends LocalS3PerfBase
{
@Override
void configureCluster(EmbeddedDruidCluster cluster)
@@ -188,9 +188,9 @@ public class S3PerfTest
}
}
- /** MinIO + Netty NIO async HTTP client. */
+ /** Local S3 container + Netty NIO async HTTP client. */
@Nested
- static class MinIO_Netty extends MinIOPerfBase
+ static class LocalS3_Netty extends LocalS3PerfBase
{
@Override
void configureCluster(EmbeddedDruidCluster cluster)
@@ -199,9 +199,9 @@ public class S3PerfTest
}
}
- /** MinIO + synchronous PutObject (transfer manager disabled). */
+ /** Local S3 container + synchronous PutObject (transfer manager disabled).
*/
@Nested
- static class MinIO_Sync extends MinIOPerfBase
+ static class LocalS3_Sync extends LocalS3PerfBase
{
@Override
void configureCluster(EmbeddedDruidCluster cluster)
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResource.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3StorageResource.java
similarity index 79%
rename from
embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResource.java
rename to
embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3StorageResource.java
index b298d6e2627..525d862abce 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResource.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3StorageResource.java
@@ -17,16 +17,18 @@
* under the License.
*/
-package org.apache.druid.testing.embedded.minio;
+package org.apache.druid.testing.embedded.s3;
import org.apache.druid.common.aws.AWSModule;
import org.apache.druid.data.input.s3.S3InputSourceConfig;
import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.metadata.DefaultPasswordProvider;
import org.apache.druid.storage.s3.S3StorageDruidModule;
import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
import org.apache.druid.testing.embedded.TestcontainerResource;
-import org.testcontainers.containers.MinIOContainer;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
@@ -41,39 +43,41 @@ import
software.amazon.awssdk.services.sts.model.Credentials;
import java.net.URI;
/**
- * A MinIO container resource for use in embedded tests as deep storage.
- * Sets up MinIO as S3-compatible storage and configures Druid's S3 connector.
+ * An S3-compatible storage container resource for use in embedded tests as
deep storage.
+ * Runs <a href="https://github.com/rustfs/rustfs";>RustFS</a> and configures
Druid's S3 connector to use it.
*/
-public class MinIOStorageResource extends TestcontainerResource<MinIOContainer>
+public class S3StorageResource extends
TestcontainerResource<GenericContainer<?>>
{
- private static final DockerImageName MINIO_IMAGE =
-
DockerImageName.parse("quay.io/minio/minio:latest").asCompatibleSubstituteFor("minio/minio");
+ private static final DockerImageName IMAGE =
DockerImageName.parse("rustfs/rustfs:1.0.0");
+ private static final int S3_PORT = 9000;
private static final String DEFAULT_BUCKET = "druid-deep-storage";
private static final String DEFAULT_BASE_KEY = "druid/segments";
- private static final String ACCESS_KEY = "minioadmin";
- private static final String SECRET_KEY = "minioadmin";
+ private static final String ACCESS_KEY = "rustfsadmin";
+ private static final String SECRET_KEY = "rustfsadmin";
private final String bucket;
private final String baseKey;
private S3Client s3Client;
- public MinIOStorageResource()
+ public S3StorageResource()
{
this(DEFAULT_BUCKET, DEFAULT_BASE_KEY);
}
- public MinIOStorageResource(String bucket, String baseKey)
+ public S3StorageResource(String bucket, String baseKey)
{
this.bucket = bucket;
this.baseKey = baseKey;
}
@Override
- protected MinIOContainer createContainer()
+ protected GenericContainer<?> createContainer()
{
- return new MinIOContainer(MINIO_IMAGE)
- .withUserName(getAccessKey())
- .withPassword(getSecretKey());
+ return new GenericContainer<>(IMAGE)
+ .withExposedPorts(S3_PORT)
+ .withEnv("RUSTFS_ACCESS_KEY", getAccessKey())
+ .withEnv("RUSTFS_SECRET_KEY", getSecretKey())
+ .waitingFor(Wait.forHttp("/health/ready").forPort(S3_PORT));
}
@Override
@@ -97,7 +101,7 @@ public class MinIOStorageResource extends
TestcontainerResource<MinIOContainer>
// Configure S3 connection properties
cluster.addCommonProperty("druid.s3.endpoint.url",
cluster.getEmbeddedHostname().useInUri(getEndpointUrl()));
- // AWS SDK v2 requires a region; use a fixed value since MinIO doesn't
validate it
+ // AWS SDK v2 requires a region; use a fixed value since the storage
container doesn't validate it
cluster.addCommonProperty("druid.s3.endpoint.signingRegion", "us-east-1");
cluster.addCommonProperty("druid.s3.accessKey", getAccessKey());
cluster.addCommonProperty("druid.s3.secretKey", getSecretKey());
@@ -129,7 +133,7 @@ public class MinIOStorageResource extends
TestcontainerResource<MinIOContainer>
public String getEndpointUrl()
{
ensureRunning();
- return getContainer().getS3URL();
+ return StringUtils.format("http://%s:%d";, getContainer().getHost(),
getContainer().getMappedPort(S3_PORT));
}
public S3Client getS3Client()
@@ -144,7 +148,7 @@ public class MinIOStorageResource extends
TestcontainerResource<MinIOContainer>
*
* @return S3InputSourceConfig with temporary credentials. The
* {@code assumeRoleArn} and {@code assumeRoleExternalId} fields are set to
null
- * since MinIO does not support them.
+ * since the storage container does not support them.
*/
public S3InputSourceConfig createTempCredentialsForInputSource()
{
@@ -152,11 +156,11 @@ public class MinIOStorageResource extends
TestcontainerResource<MinIOContainer>
final StsClient stsClient = createStsClient();
- // SDK v2 requires a non-null roleArn. MinIO does not validate the ARN,
+ // SDK v2 requires a non-null roleArn. The storage container does not
validate the ARN,
// but without an inline policy the resulting session may have no
permissions.
// An explicit S3 full-access policy ensures the temp credentials work.
final String s3FullAccessPolicy =
-
"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:*\"],\"Resource\":[\"*\"]}]}";
+
"{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:*\"],\"Resource\":[\"arn:aws:s3:::*\"]}]}";
final AssumeRoleRequest assumeRoleRequest = AssumeRoleRequest.builder()
.roleArn("arn:aws:iam::000000000000:role/test-role")
.roleSessionName("test-session")
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResourceTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3StorageResourceTest.java
similarity index 84%
rename from
embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResourceTest.java
rename to
embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3StorageResourceTest.java
index 0cdcb4b2b9a..6e865f0b0ba 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageResourceTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3StorageResourceTest.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.druid.testing.embedded.minio;
+package org.apache.druid.testing.embedded.s3;
import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
import org.apache.druid.testing.embedded.EmbeddedHostname;
@@ -31,13 +31,13 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.verify;
-public class MinIOStorageResourceTest
+public class S3StorageResourceTest
{
@Test
@Timeout(120)
- public void testMinIOContainerLifecycle()
+ public void testContainerLifecycle()
{
- final MinIOStorageResource resource = new
MinIOStorageResource("test-bucket", "test/base");
+ final S3StorageResource resource = new S3StorageResource("test-bucket",
"test/base");
final EmbeddedDruidCluster cluster =
Mockito.mock(EmbeddedDruidCluster.class);
Mockito.when(cluster.getEmbeddedHostname()).thenReturn(EmbeddedHostname.localhost());
resource.beforeStart(cluster);
@@ -50,16 +50,16 @@ public class MinIOStorageResourceTest
// Verify container properties
assertEquals("test-bucket", resource.getBucket());
assertEquals("test/base", resource.getBaseKey());
- assertEquals("minioadmin", resource.getAccessKey());
- assertEquals("minioadmin", resource.getSecretKey());
+ assertEquals("rustfsadmin", resource.getAccessKey());
+ assertEquals("rustfsadmin", resource.getSecretKey());
// Verify all the required properties are set
verify(cluster).addCommonProperty("druid.storage.type", "s3");
verify(cluster).addCommonProperty("druid.indexer.logs.type", "s3");
verify(cluster).addCommonProperty("druid.s3.enablePathStyleAccess",
"true");
verify(cluster).addCommonProperty("druid.s3.protocol", "http");
- verify(cluster).addCommonProperty("druid.s3.accessKey", "minioadmin");
- verify(cluster).addCommonProperty("druid.s3.secretKey", "minioadmin");
+ verify(cluster).addCommonProperty("druid.s3.accessKey", "rustfsadmin");
+ verify(cluster).addCommonProperty("druid.s3.secretKey", "rustfsadmin");
verify(cluster).addCommonProperty("druid.storage.bucket", "test-bucket");
verify(cluster).addCommonProperty("druid.storage.baseKey", "test/base");
verify(cluster).addCommonProperty("druid.indexer.logs.s3Bucket",
"test-bucket");
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3StorageTest.java
similarity index 84%
rename from
embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageTest.java
rename to
embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3StorageTest.java
index 199ddaec9c7..3d80091c8d9 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/MinIOStorageTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3StorageTest.java
@@ -17,23 +17,23 @@
* under the License.
*/
-package org.apache.druid.testing.embedded.minio;
+package org.apache.druid.testing.embedded.s3;
import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
import org.apache.druid.testing.embedded.EmbeddedRouter;
import org.apache.druid.testing.embedded.indexing.IndexTaskTest;
/**
- * Same as {@link IndexTaskTest}, but using a MinIO metadata store through the
S3 extension.
+ * Same as {@link IndexTaskTest}, but using S3 deep storage (an S3-compatible
testcontainer) through the S3 extension.
*/
-public class MinIOStorageTest extends IndexTaskTest
+public class S3StorageTest extends IndexTaskTest
{
@Override
public EmbeddedDruidCluster createCluster()
{
return EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper()
.useLatchableEmitter()
- .addResource(new MinIOStorageResource())
+ .addResource(new S3StorageResource())
.addServer(overlord)
.addServer(coordinator)
.addServer(indexer)
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/S3TestUtil.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3TestUtil.java
similarity index 99%
rename from
embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/S3TestUtil.java
rename to
embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3TestUtil.java
index 99672275d4f..e2996c8d2ff 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/minio/S3TestUtil.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3TestUtil.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.druid.testing.embedded.minio;
+package org.apache.druid.testing.embedded.s3;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.testing.embedded.indexing.Resources;
diff --git a/extensions-core/s3-extensions/pom.xml
b/extensions-core/s3-extensions/pom.xml
index 9b6581e8434..c872293a47c 100644
--- a/extensions-core/s3-extensions/pom.xml
+++ b/extensions-core/s3-extensions/pom.xml
@@ -231,12 +231,6 @@
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.testcontainers</groupId>
- <artifactId>testcontainers-minio</artifactId>
- <version>${testcontainers.version}</version>
- <scope>test</scope>
- </dependency>
</dependencies>
<profiles>
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/RustFSContainer.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/RustFSContainer.java
new file mode 100644
index 00000000000..9a658e62abf
--- /dev/null
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/RustFSContainer.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.storage.s3;
+
+import org.apache.druid.java.util.common.StringUtils;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.DockerImageName;
+
+/**
+ * Testcontainer for <a href="https://github.com/rustfs/rustfs";>RustFS</a>, an
S3-compatible object store, for tests
+ * that need a real S3 endpoint.
+ */
+public class RustFSContainer extends GenericContainer<RustFSContainer>
+{
+ public static final DockerImageName IMAGE =
DockerImageName.parse("rustfs/rustfs:1.0.0");
+ public static final String DEFAULT_ACCESS_KEY = "rustfsadmin";
+ public static final String DEFAULT_SECRET_KEY = "rustfsadmin";
+
+ private static final int S3_PORT = 9000;
+
+ private String accessKey = DEFAULT_ACCESS_KEY;
+ private String secretKey = DEFAULT_SECRET_KEY;
+
+ public RustFSContainer()
+ {
+ super(IMAGE);
+ withExposedPorts(S3_PORT);
+ waitingFor(Wait.forHttp("/health/ready").forPort(S3_PORT));
+ }
+
+ public RustFSContainer withCredentials(String accessKey, String secretKey)
+ {
+ this.accessKey = accessKey;
+ this.secretKey = secretKey;
+ return this;
+ }
+
+ @Override
+ protected void configure()
+ {
+ withEnv("RUSTFS_ACCESS_KEY", accessKey);
+ withEnv("RUSTFS_SECRET_KEY", secretKey);
+ }
+
+ public String getS3URL()
+ {
+ return StringUtils.format("http://%s:%d";, getHost(),
getMappedPort(S3_PORT));
+ }
+
+ public String getAccessKey()
+ {
+ return accessKey;
+ }
+
+ public String getSecretKey()
+ {
+ return secretKey;
+ }
+}
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3MultipartUploadTest.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3MultipartUploadTest.java
index f1ec809a8c3..ce3979db5a8 100644
---
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3MultipartUploadTest.java
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3MultipartUploadTest.java
@@ -27,10 +27,8 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
-import org.testcontainers.containers.MinIOContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
-import org.testcontainers.utility.DockerImageName;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
@@ -66,10 +64,7 @@ public class ServerSideEncryptingAmazonS3MultipartUploadTest
private static final long CONFIGURED_PART_SIZE = 16 * MIB;
@Container
- private static final MinIOContainer MINIO =
- new MinIOContainer(
-
DockerImageName.parse("quay.io/minio/minio:latest").asCompatibleSubstituteFor("minio/minio")
- ).withEnv("MINIO_DOMAIN", "localhost");
+ private static final RustFSContainer S3 = new RustFSContainer();
@TempDir
public File temporaryFolder;
@@ -89,7 +84,7 @@ public class ServerSideEncryptingAmazonS3MultipartUploadTest
@Override
public String getUrl()
{
- return MINIO.getS3URL();
+ return S3.getS3URL();
}
@Override
@@ -99,7 +94,7 @@ public class ServerSideEncryptingAmazonS3MultipartUploadTest
}
};
- // MinIO is reached by host:port, so bucket-as-subdomain addressing will
not resolve.
+ // The container is reached by host:port, so bucket-as-subdomain
addressing will not resolve.
final AWSClientConfig clientConfig = new AWSClientConfig()
{
@Override
@@ -110,7 +105,7 @@ public class ServerSideEncryptingAmazonS3MultipartUploadTest
};
s3 = ServerSideEncryptingAmazonS3.builder(
-
StaticCredentialsProvider.create(AwsBasicCredentials.create(MINIO.getUserName(),
MINIO.getPassword())),
+
StaticCredentialsProvider.create(AwsBasicCredentials.create(S3.getAccessKey(),
S3.getSecretKey())),
new S3StorageConfig(new NoopServerSideEncryption(), transferConfig),
null,
endpointConfig,
diff --git
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java
index 61c1bbdb7f9..cd418bbb56c 100644
---
a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java
+++
b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java
@@ -27,18 +27,16 @@ import
org.apache.druid.java.util.metrics.StubServiceEmitter;
import org.apache.druid.query.DruidProcessingConfigTest;
import org.apache.druid.storage.StorageConnector;
import org.apache.druid.storage.remote.ChunkingStorageConnectorParameters;
+import org.apache.druid.storage.s3.RustFSContainer;
import org.apache.druid.storage.s3.ServerSideEncryptingAmazonS3;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
-import org.testcontainers.containers.MinIOContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
-import org.testcontainers.utility.DockerImageName;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
@@ -69,7 +67,7 @@ public class S3StorageConnectorTest
private static final String PREFIX = "P/R/E/F/I/X";
public static final String TEST_FILE = "test.csv";
@Container
- private static final MinIOContainer MINIO = MinioUtil.createContainer();
+ private static final RustFSContainer S3 = new RustFSContainer();
@TempDir
public static File temporaryFolder;
private ServerSideEncryptingAmazonS3 s3Client;
@@ -79,7 +77,7 @@ public class S3StorageConnectorTest
@BeforeEach
public void setup() throws IOException
{
- s3Client = MinioUtil.createS3Client(MINIO);
+ s3Client = S3ClientUtil.createS3Client(S3);
try {
s3Client.getS3Client().headBucket(builder -> builder.bucket(BUCKET));
}
@@ -141,7 +139,7 @@ public class S3StorageConnectorTest
);
StorageConnector unauthorizedStorageConnector = new S3StorageConnector(
s3OutputConfig,
- MinioUtil.createUnauthorizedS3Client(MINIO),
+ S3ClientUtil.createUnauthorizedS3Client(S3),
new S3UploadManager(
s3OutputConfig,
new S3ExportConfig("tempDir", new HumanReadableBytes("5MiB"), 1,
null),
@@ -339,58 +337,29 @@ public class S3StorageConnectorTest
Assertions.assertEquals(ImmutableList.of("listFirst"), listDirResult);
}
- // adapted from apache iceberg tests
- private static class MinioUtil
+ private static class S3ClientUtil
{
- private MinioUtil()
+ private S3ClientUtil()
{
}
- public static MinIOContainer createContainer()
+ public static ServerSideEncryptingAmazonS3 createS3Client(RustFSContainer
container)
{
- return createContainer(null);
+ return createS3Client(container, container.getSecretKey());
}
- public static MinIOContainer createContainer(AwsCredentials credentials)
+ public static ServerSideEncryptingAmazonS3
createUnauthorizedS3Client(RustFSContainer container)
{
- MinIOContainer container = new MinIOContainer(
-
DockerImageName.parse("quay.io/minio/minio:latest").asCompatibleSubstituteFor("minio/minio")
- );
-
- // this enables virtual-host-style requests. see
- // https://github.com/minio/minio/tree/master/docs/config#domain
- container.withEnv("MINIO_DOMAIN", "localhost");
-
- if (credentials != null) {
- container.withUserName(credentials.accessKeyId());
- container.withPassword(credentials.secretAccessKey());
- }
-
- return container;
- }
-
- public static ServerSideEncryptingAmazonS3 createS3Client(MinIOContainer
container)
- {
- final S3Client s3Client = S3Client.builder()
- .endpointOverride(URI.create(container.getS3URL()))
- .region(Region.US_EAST_1)
- .credentialsProvider(StaticCredentialsProvider.create(
- AwsBasicCredentials.create(container.getUserName(),
container.getPassword())))
- .forcePathStyle(true) // OSX won't resolve subdomains
- .build();
-
- return ServerSideEncryptingAmazonS3.builder()
- .setS3ClientSupplier(() -> s3Client)
- .build();
+ return createS3Client(container, "wrong");
}
- public static ServerSideEncryptingAmazonS3
createUnauthorizedS3Client(MinIOContainer container)
+ private static ServerSideEncryptingAmazonS3 createS3Client(RustFSContainer
container, String secretKey)
{
final S3Client s3Client = S3Client.builder()
.endpointOverride(URI.create(container.getS3URL()))
.region(Region.US_EAST_1)
.credentialsProvider(StaticCredentialsProvider.create(
- AwsBasicCredentials.create(container.getUserName(), "wrong")))
+ AwsBasicCredentials.create(container.getAccessKey(), secretKey)))
.forcePathStyle(true) // OSX won't resolve subdomains
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
