This is an automated email from the ASF dual-hosted git repository.
milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 2da1bf6ed ci: run the S3 integration tests against RustFS instead of
MinIO (#2488)
2da1bf6ed is described below
commit 2da1bf6ed1119c925c7fe568147b21cfb211bfea
Author: NoahKusaba <[email protected]>
AuthorDate: Thu Sep 24 11:05:08 2026 -0400
ci: run the S3 integration tests against RustFS instead of MinIO (#2488)
MinIO withdrew its minio/minio image from Docker Hub on 2026-09-11 and from
quay.io on 2026-09-24, so the object_store tests can no longer pull it and
fail on every run. Use RustFS, an S3-compatible server published on Docker
Hub, pinned to 1.0.0.
testcontainers-modules was only here for its MinIO module, so depend on
testcontainers directly and start a GenericImage, waiting on RustFS's
/health endpoint since it logs only warnings by default. The bucket is still
created with mkdir under the data volume, which RustFS serves as a bucket.
custom-client's instructions start RustFS too, and now create the bucket the
example writes to.
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
Cargo.lock | 12 ++-------
examples/Cargo.toml | 2 +-
examples/examples/custom-client.rs | 13 +++++-----
examples/tests/common/mod.rs | 53 +++++++++++++++++++++-----------------
examples/tests/object_store.rs | 53 ++++++++++++++++++++++++--------------
5 files changed, 73 insertions(+), 60 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index 74fc48ba6..e6790ad16 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1167,7 +1167,7 @@ dependencies = [
"object_store",
"rustls",
"tempfile",
- "testcontainers-modules",
+ "testcontainers",
"tokio",
"tonic",
"url",
@@ -7105,6 +7105,7 @@ dependencies = [
"memchr",
"parse-display",
"pin-project-lite",
+ "reqwest 0.13.5",
"serde",
"serde_json",
"serde_with",
@@ -7115,15 +7116,6 @@ dependencies = [
"url",
]
-[[package]]
-name = "testcontainers-modules"
-version = "0.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e5985fde5befe4ffa77a052e035e16c2da86e8bae301baa9f9904ad3c494d357"
-dependencies = [
- "testcontainers",
-]
-
[[package]]
name = "textwrap"
version = "0.11.0"
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index 447a26a41..0e573c4ce 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -70,7 +70,7 @@ futures = { workspace = true }
log = { workspace = true }
object_store = { workspace = true, features = ["aws"] }
tempfile = { workspace = true }
-testcontainers-modules = { version = "0.15", features = ["minio"] }
+testcontainers = { version = "0.27", features = ["http_wait_plain"] }
tokio = { workspace = true, features = [
"macros",
"rt",
diff --git a/examples/examples/custom-client.rs
b/examples/examples/custom-client.rs
index 430d20ae2..619e5a83e 100644
--- a/examples/examples/custom-client.rs
+++ b/examples/examples/custom-client.rs
@@ -23,23 +23,24 @@ use datafusion::{assert_batches_eq,
prelude::SessionContext};
/// bucket name to be used for this example
const S3_BUCKET: &str = "ballista";
/// S3 access key
-const S3_ACCESS_KEY_ID: &str = "MINIO";
+const S3_ACCESS_KEY_ID: &str = "BALLISTA";
/// S3 secret key
-const S3_SECRET_KEY: &str = "MINIOSECRET";
+const S3_SECRET_KEY: &str = "BALLISTASECRET";
///
/// # Extending Ballista
///
/// This example demonstrates how to extend ballista scheduler and executor
registering new object store registry.
-/// It uses local [minio](https://min.io) to act as S3 object store.
+/// It uses a local [RustFS](https://github.com/rustfs/rustfs) server to act
as S3 object store.
///
/// Ballista will be extended providing custom session configuration, runtime
environment and session state.
///
-/// Minio can be started:
+/// RustFS can be started, along with the bucket this example writes to:
///
/// ```bash
-/// docker run --rm -p 9000:9000 -p 9001:9001 -e "MINIO_ACCESS_KEY=MINIO" -e
"MINIO_SECRET_KEY=MINIOSECRET" quay.io/minio/minio server /data
--console-address ":9001"
+/// docker run --rm -d --name rustfs -p 9000:9000 -e
RUSTFS_ACCESS_KEY=BALLISTA -e RUSTFS_SECRET_KEY=BALLISTASECRET
rustfs/rustfs:1.0.0
+/// docker exec rustfs mkdir /data/ballista
/// ```
-/// After minio, we need to start `custom-scheduler`
+/// After RustFS, we need to start `custom-scheduler`
///
/// ```bash
/// cargo run --example custom-scheduler
diff --git a/examples/tests/common/mod.rs b/examples/tests/common/mod.rs
index 32c4b5d0c..48704cb34 100644
--- a/examples/tests/common/mod.rs
+++ b/examples/tests/common/mod.rs
@@ -24,23 +24,25 @@ use ballista_scheduler::SessionBuilder;
use datafusion::execution::SessionState;
use datafusion::prelude::SessionConfig;
use object_store::aws::AmazonS3Builder;
-use testcontainers_modules::minio::MinIO;
-use testcontainers_modules::testcontainers::ContainerRequest;
-use testcontainers_modules::testcontainers::core::{CmdWaitFor, ExecCommand};
-use testcontainers_modules::{minio, testcontainers::ImageExt};
+use testcontainers::core::wait::HttpWaitStrategy;
+use testcontainers::core::{CmdWaitFor, ContainerPort, ExecCommand, WaitFor};
+use testcontainers::{ContainerRequest, GenericImage, ImageExt};
pub const REGION: &str = "eu-west-1";
pub const BUCKET: &str = "ballista";
-pub const ACCESS_KEY_ID: &str = "MINIO";
-pub const SECRET_KEY: &str = "MINIOMINIO";
+pub const ACCESS_KEY_ID: &str = "BALLISTA";
+pub const SECRET_KEY: &str = "BALLISTASECRET";
-/// Registry override for the image pinned by `testcontainers-modules`.
+/// S3-compatible server the integration tests run against.
///
-/// MinIO withdrew `minio/minio` from Docker Hub on 2026-09-11, so the image
the
-/// crate pins no longer resolves and every test that starts a container
panics.
-/// quay.io still serves the same tag, so only the registry changes here.
-/// See <https://github.com/apache/datafusion/issues/25215>.
-const MINIO_IMAGE_NAME: &str = "quay.io/minio/minio";
+/// These tests used MinIO until it withdrew its `minio/minio` image from
Docker Hub
+/// (2026-09-11) and then quay.io (2026-09-24). RustFS is an S3-compatible
server
+/// published on Docker Hub; see <https://github.com/rustfs/rustfs>.
+const RUSTFS_IMAGE: &str = "rustfs/rustfs";
+const RUSTFS_TAG: &str = "1.0.0";
+
+/// Port the S3 API listens on inside the container.
+pub const S3_PORT: u16 = 9000;
#[allow(dead_code)]
pub fn create_s3_store(
@@ -59,26 +61,29 @@ pub fn create_s3_store(
}
#[allow(dead_code)]
-pub fn create_minio_container() -> ContainerRequest<minio::MinIO> {
- MinIO::default()
- .with_name(MINIO_IMAGE_NAME)
- .with_env_var("MINIO_ACCESS_KEY", ACCESS_KEY_ID)
- .with_env_var("MINIO_SECRET_KEY", SECRET_KEY)
+pub fn create_s3_container() -> ContainerRequest<GenericImage> {
+ // RustFS logs only warnings by default, so wait on its health endpoint
+ // rather than a startup message.
+ let ready = HttpWaitStrategy::new("/health")
+ .with_port(ContainerPort::Tcp(S3_PORT))
+ .with_expected_status_code(200u16);
+
+ GenericImage::new(RUSTFS_IMAGE, RUSTFS_TAG)
+ .with_exposed_port(ContainerPort::Tcp(S3_PORT))
+ .with_wait_for(WaitFor::http(ready))
+ .with_env_var("RUSTFS_ACCESS_KEY", ACCESS_KEY_ID)
+ .with_env_var("RUSTFS_SECRET_KEY", SECRET_KEY)
}
#[allow(dead_code)]
pub fn create_bucket_command() -> ExecCommand {
- // this is hack to create a bucket without creating s3 client.
- // this works with current testcontainer (and image) version
'RELEASE.2022-02-07T08-17-33Z'.
- // (testcontainer does not await properly on latest image version)
- //
- // if testcontainer image version change to something newer we should use
"mc mb /data/ballista"
- // to crate a bucket.
+ // A top-level directory under the data volume is a bucket, which avoids
+ // pulling an S3 client in just to create one.
ExecCommand::new(vec![
"mkdir".to_string(),
format!("/data/{}", crate::common::BUCKET),
])
- .with_cmd_ready_condition(CmdWaitFor::seconds(1))
+ .with_cmd_ready_condition(CmdWaitFor::exit_code(0))
}
/// starts a ballista cluster for integration tests
diff --git a/examples/tests/object_store.rs b/examples/tests/object_store.rs
index 6202bb8f9..6db234fba 100644
--- a/examples/tests/object_store.rs
+++ b/examples/tests/object_store.rs
@@ -19,7 +19,7 @@
//!
//! Tests demonstrate how to setup object stores with ballista.
//!
-//! Test depend on Minio testcontainer acting as S3 object
+//! Test depend on a RustFS testcontainer acting as S3 object
//! store.
//!
//! Tesctoncainers require docker to run.
@@ -36,11 +36,11 @@ mod standalone {
use datafusion::{assert_batches_eq, prelude::SessionContext};
use datafusion::{error::DataFusionError, execution::SessionStateBuilder};
use std::sync::Arc;
- use testcontainers_modules::testcontainers::runners::AsyncRunner;
+ use testcontainers::runners::AsyncRunner;
#[tokio::test]
async fn should_execute_sql_write() -> datafusion::error::Result<()> {
- let container = crate::common::create_minio_container();
+ let container = crate::common::create_s3_container();
let node = container.start().await.unwrap();
node.exec(crate::common::create_bucket_command())
@@ -48,7 +48,10 @@ mod standalone {
.unwrap();
let host = node.get_host().await.unwrap();
- let port = node.get_host_port_ipv4(9000).await.unwrap();
+ let port = node
+ .get_host_port_ipv4(crate::common::S3_PORT)
+ .await
+ .unwrap();
let object_store = crate::common::create_s3_store(&host.to_string(),
port)
.map_err(|e| DataFusionError::External(e.into()))?;
@@ -133,13 +136,13 @@ mod remote {
use datafusion::{assert_batches_eq, prelude::SessionContext};
use datafusion::{error::DataFusionError, execution::SessionStateBuilder};
use std::sync::Arc;
- use testcontainers_modules::testcontainers::runners::AsyncRunner;
+ use testcontainers::runners::AsyncRunner;
#[tokio::test]
async fn should_execute_sql_write() -> datafusion::error::Result<()> {
let test_data = examples_test_data();
- let container = crate::common::create_minio_container();
+ let container = crate::common::create_s3_container();
let node = container.start().await.unwrap();
node.exec(crate::common::create_bucket_command())
@@ -147,7 +150,10 @@ mod remote {
.unwrap();
let host = node.get_host().await.unwrap();
- let port = node.get_host_port_ipv4(9000).await.unwrap();
+ let port = node
+ .get_host_port_ipv4(crate::common::S3_PORT)
+ .await
+ .unwrap();
let object_store = crate::common::create_s3_store(&host.to_string(),
port)
.map_err(|e| DataFusionError::External(e.into()))?;
@@ -230,7 +236,7 @@ mod custom_s3_config {
use datafusion::{assert_batches_eq, prelude::SessionContext};
use datafusion::{error::DataFusionError, execution::SessionStateBuilder};
use std::sync::Arc;
- use testcontainers_modules::testcontainers::runners::AsyncRunner;
+ use testcontainers::runners::AsyncRunner;
#[tokio::test]
async fn should_configure_s3_execute_sql_write_remote()
@@ -238,9 +244,9 @@ mod custom_s3_config {
let test_data = examples_test_data();
//
- // Minio cluster setup
+ // S3 (RustFS) container setup
//
- let container = crate::common::create_minio_container();
+ let container = crate::common::create_s3_container();
let node = container.start().await.unwrap();
node.exec(crate::common::create_bucket_command())
@@ -248,10 +254,13 @@ mod custom_s3_config {
.unwrap();
let endpoint_host = node.get_host().await.unwrap();
- let endpoint_port = node.get_host_port_ipv4(9000).await.unwrap();
+ let endpoint_port = node
+ .get_host_port_ipv4(crate::common::S3_PORT)
+ .await
+ .unwrap();
log::info!(
- "MINIO testcontainers host: {}, port: {}",
+ "S3 testcontainers host: {}, port: {}",
endpoint_host,
endpoint_port
);
@@ -377,9 +386,9 @@ mod custom_s3_config {
let test_data = examples_test_data();
//
- // Minio cluster setup
+ // S3 (RustFS) container setup
//
- let container = crate::common::create_minio_container();
+ let container = crate::common::create_s3_container();
let node = container.start().await.unwrap();
node.exec(crate::common::create_bucket_command())
@@ -387,10 +396,13 @@ mod custom_s3_config {
.unwrap();
let endpoint_host = node.get_host().await.unwrap();
- let endpoint_port = node.get_host_port_ipv4(9000).await.unwrap();
+ let endpoint_port = node
+ .get_host_port_ipv4(crate::common::S3_PORT)
+ .await
+ .unwrap();
log::info!(
- "MINIO testcontainers host: {}, port: {}",
+ "S3 testcontainers host: {}, port: {}",
endpoint_host,
endpoint_port
);
@@ -526,9 +538,9 @@ mod custom_s3_config {
let test_data = examples_test_data();
//
- // Minio cluster setup
+ // S3 (RustFS) container setup
//
- let container = crate::common::create_minio_container();
+ let container = crate::common::create_s3_container();
let node = container.start().await.unwrap();
node.exec(crate::common::create_bucket_command())
@@ -536,7 +548,10 @@ mod custom_s3_config {
.unwrap();
let endpoint_host = node.get_host().await.unwrap();
- let endpoint_port = node.get_host_port_ipv4(9000).await.unwrap();
+ let endpoint_port = node
+ .get_host_port_ipv4(crate::common::S3_PORT)
+ .await
+ .unwrap();
//
// Session Context and Ballista cluster setup
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
