This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 450863e7 ci: replace MinIO with RustFS in S3 tests (#962)
450863e7 is described below
commit 450863e776aa13d3ef3e71bf44af30f07405efc8
Author: Sreesh Maheshwar <[email protected]>
AuthorDate: Fri Sep 25 03:01:56 2026 +0100
ci: replace MinIO with RustFS in S3 tests (#962)
* ci: replace MinIO with RustFS in S3 tests
Use RustFS 1.0.0 for Docker and native test fixtures, and initialize the S3
bucket with the AWS CLI.
Generated-by: Codex
* ci: improve RustFS startup diagnostics and readiness
Expose server logs, retry bucket initialization, verify native downloads,
and disable the unused console.
Generated-by: Codex
* ci: use generic object store naming and improve startup diagnostics
- Rename the CI step to "Start Object Store"
- Enable shell tracing in start_object_store.sh
- Print the native server log only when it was started natively
- Replace MinIO-specific wording in S3 comments
Co-authored-by: Copilot App <[email protected]>
---------
Co-authored-by: Kevin Liu <[email protected]>
Co-authored-by: Copilot App <[email protected]>
---
.github/workflows/aws_test.yml | 8 +-
ci/scripts/start_minio.sh | 158 -------------------------------
ci/scripts/start_object_store.sh | 134 ++++++++++++++++++++++++++
src/iceberg/arrow/s3/arrow_s3_file_io.cc | 2 +-
src/iceberg/arrow/s3/s3_properties.h | 4 +-
5 files changed, 141 insertions(+), 165 deletions(-)
diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml
index e5918561..9214027a 100644
--- a/.github/workflows/aws_test.yml
+++ b/.github/workflows/aws_test.yml
@@ -74,8 +74,8 @@ jobs:
bundle_awssdk: "ON"
env:
ICEBERG_TEST_S3_URI: s3://iceberg-test
- AWS_ACCESS_KEY_ID: minio
- AWS_SECRET_ACCESS_KEY: minio123
+ AWS_ACCESS_KEY_ID: admin
+ AWS_SECRET_ACCESS_KEY: password
AWS_DEFAULT_REGION: us-east-1
AWS_ENDPOINT_URL: http://127.0.0.1:9000
AWS_EC2_METADATA_DISABLED: "TRUE"
@@ -114,10 +114,10 @@ jobs:
run: |
echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV
echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV
- - name: Start MinIO
+ - name: Start Object Store
if: ${{ matrix.s3 == 'ON' }}
shell: bash
- run: bash ci/scripts/start_minio.sh
+ run: bash ci/scripts/start_object_store.sh
- name: Set up sccache
uses: ./.github/actions/setup-sccache
with:
diff --git a/ci/scripts/start_minio.sh b/ci/scripts/start_minio.sh
deleted file mode 100644
index 793c7852..00000000
--- a/ci/scripts/start_minio.sh
+++ /dev/null
@@ -1,158 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -eux
-
-MINIO_ROOT_USER="${MINIO_ROOT_USER:-minio}"
-MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD:-minio123}"
-MINIO_IMAGE="${MINIO_IMAGE:-quay.io/minio/minio:latest}"
-MINIO_CONTAINER_NAME="${MINIO_CONTAINER_NAME:-iceberg-minio}"
-MINIO_PORT="${MINIO_PORT:-9000}"
-MINIO_CONSOLE_PORT="${MINIO_CONSOLE_PORT:-9001}"
-MINIO_BUCKET="${MINIO_BUCKET:-iceberg-test}"
-MINIO_ENDPOINT="${MINIO_ENDPOINT:-http://127.0.0.1:${MINIO_PORT}}";
-
-wait_for_minio() {
- for i in {1..30}; do
- if curl -fsS "${MINIO_ENDPOINT}/minio/health/ready" >/dev/null; then
- return 0
- fi
- sleep 1
- done
- echo "MinIO did not become ready after 30 seconds." >&2
- echo "Endpoint: ${MINIO_ENDPOINT}" >&2
- if command -v docker >/dev/null 2>&1; then
- docker logs "${MINIO_CONTAINER_NAME}" 2>&1 || true
- fi
- return 1
-}
-
-start_minio_docker() {
- if ! command -v docker >/dev/null 2>&1; then
- return 1
- fi
-
- if docker ps -a --format '{{.Names}}' | grep -q
"^${MINIO_CONTAINER_NAME}\$"; then
- docker rm -f "${MINIO_CONTAINER_NAME}"
- fi
-
- docker run -d --name "${MINIO_CONTAINER_NAME}" \
- -p "${MINIO_PORT}:9000" -p "${MINIO_CONSOLE_PORT}:9001" \
- -e "MINIO_ROOT_USER=${MINIO_ROOT_USER}" \
- -e "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}" \
- "${MINIO_IMAGE}" \
- server /data --console-address ":${MINIO_CONSOLE_PORT}"
-
- wait_for_minio
-}
-
-start_minio_macos() {
- if ! command -v brew >/dev/null 2>&1; then
- echo "brew is required to start MinIO on macOS without Docker" >&2
- return 1
- fi
-
- brew install minio
- MINIO_ROOT_USER="${MINIO_ROOT_USER}"
MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD}" \
- minio server /tmp/minio --console-address ":${MINIO_CONSOLE_PORT}" &
- wait_for_minio
-}
-
-download_mc() {
- local uname_out
- uname_out="$(uname -s)"
- local mc_release="RELEASE.2025-08-13T08-35-41Z"
- local
mc_download_url="https://github.com/minio/mc/releases/download/${mc_release}";
-
- local mc_dir
- mc_dir="${RUNNER_TEMP:-/tmp}"
- mkdir -p "${mc_dir}"
-
- case "${uname_out}" in
- Linux*)
- MC_BIN="${mc_dir}/mc"
- curl -fsSL "${mc_download_url}/mc.linux-amd64.${mc_release}" -o
"${MC_BIN}"
- chmod +x "${MC_BIN}"
- ;;
- Darwin*)
- MC_BIN="${mc_dir}/mc"
- local arch
- arch="$(uname -m)"
- if [ "${arch}" = "arm64" ]; then
- curl -fsSL "${mc_download_url}/mc.darwin-arm64.${mc_release}" -o
"${MC_BIN}"
- else
- curl -fsSL "${mc_download_url}/mc.darwin-amd64.${mc_release}" -o
"${MC_BIN}"
- fi
- chmod +x "${MC_BIN}"
- ;;
- MINGW*|MSYS*|CYGWIN*)
- MC_BIN="${mc_dir}/mc.exe"
- curl -fsSL "${mc_download_url}/mc.windows-amd64.${mc_release}.exe" -o
"${MC_BIN}"
- ;;
- *)
- echo "Unsupported OS for mc: ${uname_out}" >&2
- return 1
- ;;
- esac
-}
-
-create_bucket() {
- download_mc
- for i in {1..30}; do
- if "${MC_BIN}" alias set local "${MINIO_ENDPOINT}" "${MINIO_ROOT_USER}"
"${MINIO_ROOT_PASSWORD}"; then
- break
- fi
- sleep 1
- done
- "${MC_BIN}" mb --ignore-existing "local/${MINIO_BUCKET}"
-}
-
-start_minio_windows() {
- local minio_dir="${RUNNER_TEMP:-/tmp}"
- local minio_bin="${minio_dir}/minio.exe"
- local minio_release="RELEASE.2025-09-07T16-13-09Z"
- curl -fsSL \
-
"https://github.com/minio/minio/releases/download/${minio_release}/minio.windows-amd64.${minio_release}.exe";
\
- -o "${minio_bin}"
- MINIO_ROOT_USER="${MINIO_ROOT_USER}"
MINIO_ROOT_PASSWORD="${MINIO_ROOT_PASSWORD}" \
- "${minio_bin}" server "${minio_dir}/minio-data" --console-address
":${MINIO_CONSOLE_PORT}" &
- wait_for_minio
-}
-
-case "$(uname -s)" in
- Darwin*)
- if ! start_minio_docker; then
- start_minio_macos
- fi
- ;;
- MINGW*|MSYS*|CYGWIN*)
- if ! start_minio_docker; then
- start_minio_windows
- fi
- ;;
- Linux*)
- start_minio_docker
- ;;
- *)
- echo "Unsupported OS: $(uname -s)" >&2
- exit 1
- ;;
-esac
-
-create_bucket
diff --git a/ci/scripts/start_object_store.sh b/ci/scripts/start_object_store.sh
new file mode 100644
index 00000000..dfd82514
--- /dev/null
+++ b/ci/scripts/start_object_store.sh
@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euxo pipefail
+
+OBJECT_STORE_VERSION="${OBJECT_STORE_VERSION:-1.0.0}"
+OBJECT_STORE_IMAGE="${OBJECT_STORE_IMAGE:-rustfs/rustfs:${OBJECT_STORE_VERSION}}"
+OBJECT_STORE_CONTAINER_NAME="${OBJECT_STORE_CONTAINER_NAME:-iceberg-object-store}"
+OBJECT_STORE_ACCESS_KEY="${AWS_ACCESS_KEY_ID:-admin}"
+OBJECT_STORE_SECRET_KEY="${AWS_SECRET_ACCESS_KEY:-password}"
+OBJECT_STORE_PORT="${OBJECT_STORE_PORT:-9000}"
+OBJECT_STORE_BUCKET="${OBJECT_STORE_BUCKET:-iceberg-test}"
+OBJECT_STORE_ENDPOINT="${AWS_ENDPOINT_URL:-http://127.0.0.1:${OBJECT_STORE_PORT}}";
+OBJECT_STORE_DIR="${RUNNER_TEMP:-/tmp}/iceberg-object-store"
+OBJECT_STORE_LOG=""
+
+wait_for_object_store() {
+ for ((attempt = 0; attempt < 60; attempt++)); do
+ if curl -fs --connect-timeout 1 --max-time 2
"${OBJECT_STORE_ENDPOINT}/health/ready" >/dev/null; then
+ return 0
+ fi
+ sleep 1
+ done
+ echo "Object store did not become ready at ${OBJECT_STORE_ENDPOINT}." >&2
+ if [ -n "${OBJECT_STORE_LOG}" ]; then
+ cat "${OBJECT_STORE_LOG}" >&2
+ else
+ docker logs "${OBJECT_STORE_CONTAINER_NAME}" >&2 || true
+ fi
+ return 1
+}
+
+start_object_store_docker() {
+ if docker container inspect "${OBJECT_STORE_CONTAINER_NAME}" >/dev/null
2>&1; then
+ docker rm -f "${OBJECT_STORE_CONTAINER_NAME}"
+ fi
+
+ docker run -d --name "${OBJECT_STORE_CONTAINER_NAME}" \
+ -p "${OBJECT_STORE_PORT}:9000" \
+ -e "RUSTFS_ACCESS_KEY=${OBJECT_STORE_ACCESS_KEY}" \
+ -e "RUSTFS_SECRET_KEY=${OBJECT_STORE_SECRET_KEY}" \
+ -e RUSTFS_CONSOLE_ENABLE=false \
+ -e RUSTFS_OBS_LOG_STDOUT_ENABLED=true \
+ "${OBJECT_STORE_IMAGE}" /data
+}
+
+start_object_store_native() {
+ local platform binary
+ case "$(uname -s)-$(uname -m)" in
+ Darwin-arm64)
+ platform=macos-aarch64
+ binary=rustfs
+ ;;
+ MINGW*-x86_64|MSYS*-x86_64|CYGWIN*-x86_64)
+ platform=windows-x86_64
+ binary=rustfs.exe
+ ;;
+ *)
+ echo "Use Docker to run RustFS on $(uname -s)-$(uname -m)." >&2
+ return 1
+ ;;
+ esac
+
+ local archive="rustfs-${platform}-v${OBJECT_STORE_VERSION}.zip"
+ local
release_url="https://github.com/rustfs/rustfs/releases/download/${OBJECT_STORE_VERSION}";
+ mkdir -p "${OBJECT_STORE_DIR}/data"
+ curl -fsSL --retry 3 \
+ "${release_url}/${archive}" -o "${OBJECT_STORE_DIR}/${archive}"
+ curl -fsSL --retry 3 "${release_url}/SHA256SUMS" -o
"${OBJECT_STORE_DIR}/SHA256SUMS"
+ (
+ cd "${OBJECT_STORE_DIR}"
+ grep -F " ${archive}" SHA256SUMS > rustfs.sha256
+ if command -v sha256sum >/dev/null 2>&1; then
+ sha256sum --check rustfs.sha256
+ else
+ shasum -a 256 --check rustfs.sha256
+ fi
+ )
+ unzip -o "${OBJECT_STORE_DIR}/${archive}" -d "${OBJECT_STORE_DIR}"
+ chmod +x "${OBJECT_STORE_DIR}/${binary}"
+ OBJECT_STORE_LOG="${OBJECT_STORE_DIR}/rustfs.log"
+ RUSTFS_ACCESS_KEY="${OBJECT_STORE_ACCESS_KEY}" \
+ RUSTFS_SECRET_KEY="${OBJECT_STORE_SECRET_KEY}" \
+ RUSTFS_ADDRESS=":${OBJECT_STORE_PORT}" \
+ RUSTFS_CONSOLE_ENABLE=false \
+ "${OBJECT_STORE_DIR}/${binary}" "${OBJECT_STORE_DIR}/data" \
+ >"${OBJECT_STORE_LOG}" 2>&1 &
+}
+
+create_bucket() {
+ export AWS_ACCESS_KEY_ID="${OBJECT_STORE_ACCESS_KEY}"
+ export AWS_SECRET_ACCESS_KEY="${OBJECT_STORE_SECRET_KEY}"
+ export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-us-east-1}"
+ # The health endpoint can pass before S3 requests are accepted.
+ for ((attempt = 0; attempt < 5; attempt++)); do
+ if aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api head-bucket
--bucket "${OBJECT_STORE_BUCKET}" >/dev/null 2>&1 || \
+ aws --endpoint-url "${OBJECT_STORE_ENDPOINT}" s3api create-bucket
--bucket "${OBJECT_STORE_BUCKET}"; then
+ return 0
+ fi
+ sleep 2
+ done
+ echo "Failed to create bucket ${OBJECT_STORE_BUCKET} at
${OBJECT_STORE_ENDPOINT}." >&2
+ return 1
+}
+
+if ! command -v aws >/dev/null 2>&1; then
+ echo "AWS CLI is required to create the test bucket." >&2
+ exit 1
+fi
+
+if command -v docker >/dev/null 2>&1 && [ "$(docker info --format
'{{.OSType}}' 2>/dev/null)" = linux ]; then
+ start_object_store_docker
+else
+ start_object_store_native
+fi
+
+wait_for_object_store
+create_bucket
diff --git a/src/iceberg/arrow/s3/arrow_s3_file_io.cc
b/src/iceberg/arrow/s3/arrow_s3_file_io.cc
index 7ce0d0fd..c981759c 100644
--- a/src/iceberg/arrow/s3/arrow_s3_file_io.cc
+++ b/src/iceberg/arrow/s3/arrow_s3_file_io.cc
@@ -109,7 +109,7 @@ Result<::arrow::fs::S3Options> ConfigureS3Options(
options.region = *region;
}
- // Configure endpoint (for MinIO, LocalStack, etc.)
+ // Configure endpoint (for S3-compatible object stores)
if (const auto* endpoint = FindProperty(properties, S3Properties::kEndpoint);
endpoint != nullptr) {
options.endpoint_override = SplitEndpointScheme(*endpoint, options);
diff --git a/src/iceberg/arrow/s3/s3_properties.h
b/src/iceberg/arrow/s3/s3_properties.h
index 605c748c..50dafa56 100644
--- a/src/iceberg/arrow/s3/s3_properties.h
+++ b/src/iceberg/arrow/s3/s3_properties.h
@@ -44,9 +44,9 @@ struct S3Properties {
static constexpr std::string_view kSessionToken = "s3.session-token";
/// AWS region, standard Iceberg client property.
static constexpr std::string_view kClientRegion = "client.region";
- /// Custom endpoint override (for MinIO, LocalStack, etc.)
+ /// Custom endpoint override (for S3-compatible object stores)
static constexpr std::string_view kEndpoint = "s3.endpoint";
- /// Whether to use path-style access (needed for MinIO)
+ /// Whether to use path-style access (needed for some S3-compatible object
stores)
static constexpr std::string_view kPathStyleAccess = "s3.path-style-access";
/// Whether SSL is enabled
static constexpr std::string_view kSslEnabled = "s3.ssl.enabled";
