This is an automated email from the ASF dual-hosted git repository.
pitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new c75b82dd59 GH-40410: [C++] Skip only s3fs-tests and s3fs-module-tests
that require MinIO if MinIO is not available (#50215)
c75b82dd59 is described below
commit c75b82dd594d60b056ec7539ec2b7e8ba2aaff13
Author: Raúl Cumplido <[email protected]>
AuthorDate: Mon Jun 22 13:23:52 2026 +0200
GH-40410: [C++] Skip only s3fs-tests and s3fs-module-tests that require
MinIO if MinIO is not available (#50215)
### Rationale for this change
There are lots of S3 tests that doesn't require MinIO which we are not
being exercised on CI because we skip the full test suite. We should skip only
the tests that use MinIO if MinIO is not found but not the ones that doesn't
require MinIO.
### What changes are included in this PR?
- Remove all occurrences of `exclude_tests="arrow-s3fs-test"`
- Add MinIO to path for macOS cpp.yml builds so MinIO tests are exercised
- Add new `MinioTestEnvironment::IsAvailable` to validate whether MinIO is
available or not as part of the test suite
- `GTEST_SKIP` tests that require MinIO
### Are these changes tested?
Yes via CI and existing tests.
I've validated this locally both ways: without MinIO the non-MinIO tests
run and the MinIO dependant ones are skipped, with MinIO on PATH the full suite
runs and passes
### Are there any user-facing changes?
No
* GitHub Issue: #40410
Authored-by: Raúl Cumplido <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
.github/workflows/cpp.yml | 2 ++
.github/workflows/cpp_extra.yml | 4 +---
ci/scripts/cpp_test.sh | 5 -----
cpp/src/arrow/filesystem/s3_test_util.cc | 13 +++++++++++
cpp/src/arrow/filesystem/s3_test_util.h | 6 +++++
cpp/src/arrow/filesystem/s3fs_module_test.cc | 33 +++++++++++++++++-----------
cpp/src/arrow/filesystem/s3fs_test.cc | 5 +++++
7 files changed, 47 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index a6ac94b4c3..17cd5da22a 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -293,6 +293,8 @@ jobs:
- name: Test
shell: bash
run: |
+ # MinIO is installed on ${ARROW_HOME}/bin
+ export PATH="${ARROW_HOME}/bin:${PATH}"
sudo sysctl -w kern.coredump=1
sudo sysctl -w kern.corefile=/tmp/core.%N.%P
ulimit -c unlimited # must enable within the same shell
diff --git a/.github/workflows/cpp_extra.yml b/.github/workflows/cpp_extra.yml
index 622cf8d1a7..c763cb6f02 100644
--- a/.github/workflows/cpp_extra.yml
+++ b/.github/workflows/cpp_extra.yml
@@ -340,10 +340,8 @@ jobs:
ARROW_TEST_DATA: ${{ github.workspace }}/testing/data
PARQUET_TEST_DATA: ${{ github.workspace
}}/cpp/submodules/parquet-testing/data
run: |
- # MinIO is required
- exclude_tests="arrow-s3fs-test"
# unstable
- exclude_tests="${exclude_tests}|arrow-acero-asof-join-node-test"
+ exclude_tests="arrow-acero-asof-join-node-test"
exclude_tests="${exclude_tests}|arrow-acero-hash-join-node-test"
ctest \
--exclude-regex "${exclude_tests}" \
diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh
index 241addbfeb..18c301c5fd 100755
--- a/ci/scripts/cpp_test.sh
+++ b/ci/scripts/cpp_test.sh
@@ -50,17 +50,12 @@ fi
if ! type storage-testbench >/dev/null 2>&1; then
exclude_tests+=("arrow-gcsfs-test")
fi
-if ! type minio >/dev/null 2>&1; then
- exclude_tests+=("arrow-s3fs-test")
-fi
case "$(uname)" in
Linux)
n_jobs=$(nproc)
;;
Darwin)
n_jobs=$(sysctl -n hw.ncpu)
- # TODO: https://github.com/apache/arrow/issues/40410
- exclude_tests+=("arrow-s3fs-test")
;;
MINGW*)
n_jobs=${NUMBER_OF_PROCESSORS:-1}
diff --git a/cpp/src/arrow/filesystem/s3_test_util.cc
b/cpp/src/arrow/filesystem/s3_test_util.cc
index 0cfe038599..daaf091450 100644
--- a/cpp/src/arrow/filesystem/s3_test_util.cc
+++ b/cpp/src/arrow/filesystem/s3_test_util.cc
@@ -192,6 +192,19 @@ void MinioTestEnvironment::SetUp() {
MakeReadaheadGenerator(std::move(impl_->server_generator_),
pool->GetCapacity());
}
+bool MinioTestEnvironment::IsAvailable() {
+ if (!available_.has_value()) {
+ // If external S3-compatible service is configured there's no need to
check for Minio
+ available_ = std::getenv(kEnvConnectString) && std::getenv(kEnvAccessKey)
&&
+ std::getenv(kEnvSecretKey);
+ if (!available_.value()) {
+ ::arrow::util::Process process;
+ available_ = process.SetExecutable(kMinioExecutableName).ok();
+ }
+ }
+ return available_.value();
+}
+
Result<std::shared_ptr<MinioTestServer>> MinioTestEnvironment::GetOneServer() {
return impl_->server_generator_().result();
}
diff --git a/cpp/src/arrow/filesystem/s3_test_util.h
b/cpp/src/arrow/filesystem/s3_test_util.h
index 0a89a7a9d5..754d8a2ee6 100644
--- a/cpp/src/arrow/filesystem/s3_test_util.h
+++ b/cpp/src/arrow/filesystem/s3_test_util.h
@@ -18,6 +18,7 @@
#pragma once
#include <memory>
+#include <optional>
#include <string>
#include <utility>
@@ -74,6 +75,11 @@ class MinioTestEnvironment : public ::testing::Environment {
Result<std::shared_ptr<MinioTestServer>> GetOneServer();
+ bool IsAvailable();
+
+ private:
+ std::optional<bool> available_;
+
protected:
struct Impl;
std::unique_ptr<Impl> impl_;
diff --git a/cpp/src/arrow/filesystem/s3fs_module_test.cc
b/cpp/src/arrow/filesystem/s3fs_module_test.cc
index f056b7fe55..f07b07fb6c 100644
--- a/cpp/src/arrow/filesystem/s3fs_module_test.cc
+++ b/cpp/src/arrow/filesystem/s3fs_module_test.cc
@@ -55,6 +55,17 @@ MinioTestEnvironment* GetMinioEnv() {
return ::arrow::internal::checked_cast<MinioTestEnvironment*>(minio_env);
}
+class S3ModuleTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ if (!GetMinioEnv()->IsAvailable()) {
+ GTEST_SKIP() << "Minio executable not found, skipping tests";
+ }
+ ASSERT_OK_AND_ASSIGN(minio_, GetMinioEnv()->GetOneServer());
+ }
+ std::shared_ptr<MinioTestServer> minio_;
+};
+
class RegistrationTestEnvironment : public ::testing::Environment {
public:
void SetUp() override {
@@ -68,12 +79,10 @@ class RegistrationTestEnvironment : public
::testing::Environment {
auto* lib_env = ::testing::AddGlobalTestEnvironment(new
RegistrationTestEnvironment);
-TEST(S3Test, FromUri) {
- ASSERT_OK_AND_ASSIGN(auto minio, GetMinioEnv()->GetOneServer());
-
+TEST_F(S3ModuleTest, FromUri) {
std::string path;
- ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri("s3://" +
minio->access_key() + ":" +
- minio->secret_key() +
+ ASSERT_OK_AND_ASSIGN(auto fs, FileSystemFromUri("s3://" +
minio_->access_key() + ":" +
+ minio_->secret_key() +
"@bucket/somedir/subdir/subfile",
&path));
@@ -83,12 +92,11 @@ TEST(S3Test, FromUri) {
"&allow_bucket_creation=0&allow_bucket_deletion=0");
}
-TEST(S3Test, FromUriAndOptionsCredentials) {
- ASSERT_OK_AND_ASSIGN(auto minio, GetMinioEnv()->GetOneServer());
+TEST_F(S3ModuleTest, FromUriAndOptionsCredentials) {
std::string path;
FileSystemFactoryOptions options{
- {"access_key", std::string(minio->access_key())},
- {"secret_key", std::string(minio->secret_key())},
+ {"access_key", std::string(minio_->access_key())},
+ {"secret_key", std::string(minio_->secret_key())},
};
// Credentials supplied via options, NOT in the URI.
ASSERT_OK_AND_ASSIGN(
@@ -111,11 +119,10 @@ class NoopRetryStrategy : public S3RetryStrategy {
};
} // namespace
-TEST(S3Test, FromUriAndOptionsRetryStrategy) {
- ASSERT_OK_AND_ASSIGN(auto minio, GetMinioEnv()->GetOneServer());
+TEST_F(S3ModuleTest, FromUriAndOptionsRetryStrategy) {
FileSystemFactoryOptions options{
- {"access_key", std::string(minio->access_key())},
- {"secret_key", std::string(minio->secret_key())},
+ {"access_key", std::string(minio_->access_key())},
+ {"secret_key", std::string(minio_->secret_key())},
{"retry_strategy",
std::shared_ptr<S3RetryStrategy>(std::make_shared<NoopRetryStrategy>())},
};
diff --git a/cpp/src/arrow/filesystem/s3fs_test.cc
b/cpp/src/arrow/filesystem/s3fs_test.cc
index d513e9ca97..114701b6d4 100644
--- a/cpp/src/arrow/filesystem/s3fs_test.cc
+++ b/cpp/src/arrow/filesystem/s3fs_test.cc
@@ -192,6 +192,9 @@ class S3TestMixin : public AwsTestMixin {
public:
void SetUp() override {
AwsTestMixin::SetUp();
+ if (!GetMinioEnv(enable_tls_)->IsAvailable()) {
+ GTEST_SKIP() << "Minio executable not found, skipping tests";
+ }
// Starting the server may fail, for example if the generated port number
// was "stolen" by another process. Run a dummy S3 operation to make sure
it
@@ -624,6 +627,7 @@ class TestS3FS : public S3TestMixin {
public:
void SetUp() override {
S3TestMixin::SetUp();
+ if (IsSkipped()) return;
// Most tests will create buckets
options_.allow_bucket_creation = true;
options_.allow_bucket_deletion = true;
@@ -1777,6 +1781,7 @@ class TestS3FSGeneric : public S3TestMixin, public
GenericFileSystemTest {
public:
void SetUp() override {
S3TestMixin::SetUp();
+ if (IsSkipped()) return;
// Set up test bucket
{
Aws::S3::Model::CreateBucketRequest req;
