(arrow) branch main updated: GH-40151: [C++] Make S3 narrative test more flexible (#40144)

[apitrou]

This is an automated email from the ASF dual-hosted git repository.

apitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git


The following commit(s) were added to refs/heads/main by this push:
     new 4dc3d04ae8 GH-40151: [C++] Make S3 narrative test more flexible 
(#40144)
4dc3d04ae8 is described below

commit 4dc3d04ae84d97d02443c0cef555a46535925c2b
Author: Antoine Pitrou <anto...@python.org>
AuthorDate: Tue Feb 20 11:55:48 2024 +0100

    GH-40151: [C++] Make S3 narrative test more flexible (#40144)
    
    `arrow-s3fs-narrative-test` allows manual testing of the S3 filesystem 
implementation against an actual S3 backend such as AWS.
    
    This PR allows customizing the endpoint address, and adds a command to 
create a bucket for testing.
    
    Validated with the LocalStack S3 server.
    
    * Closes: #40151
    
    Authored-by: Antoine Pitrou <anto...@python.org>
    Signed-off-by: Antoine Pitrou <anto...@python.org>
---
 cpp/src/arrow/filesystem/s3fs_narrative_test.cc | 35 +++++++++++++++++--------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/cpp/src/arrow/filesystem/s3fs_narrative_test.cc 
b/cpp/src/arrow/filesystem/s3fs_narrative_test.cc
index f75ca4bdfd..bbb3c32ee6 100644
--- a/cpp/src/arrow/filesystem/s3fs_narrative_test.cc
+++ b/cpp/src/arrow/filesystem/s3fs_narrative_test.cc
@@ -34,6 +34,7 @@
 #include "arrow/util/logging.h"
 
 DEFINE_bool(clear, false, "delete all bucket contents");
+DEFINE_bool(create, false, "create test bucket");
 DEFINE_bool(test, false, "run narrative test against bucket");
 
 DEFINE_bool(verbose, false, "be more verbose (includes AWS warnings)");
@@ -57,8 +58,7 @@ namespace fs {
     PrintError(context_msg, _status_or_result);            \
   } while (0)
 
-std::shared_ptr<FileSystem> MakeFileSystem() {
-  std::shared_ptr<S3FileSystem> s3fs;
+Result<std::shared_ptr<FileSystem>> MakeRootFileSystem() {
   S3Options options;
   if (!FLAGS_access_key.empty()) {
     options = S3Options::FromAccessKey(FLAGS_access_key, FLAGS_secret_key);
@@ -68,8 +68,13 @@ std::shared_ptr<FileSystem> MakeFileSystem() {
   options.endpoint_override = FLAGS_endpoint;
   options.scheme = FLAGS_scheme;
   options.region = FLAGS_region;
-  s3fs = S3FileSystem::Make(options).ValueOrDie();
-  return std::make_shared<SubTreeFileSystem>(FLAGS_bucket, s3fs);
+  options.allow_bucket_creation = FLAGS_create;
+  return S3FileSystem::Make(options);
+}
+
+Result<std::shared_ptr<FileSystem>> MakeFileSystem() {
+  ARROW_ASSIGN_OR_RAISE(auto fs, MakeRootFileSystem());
+  return std::make_shared<SubTreeFileSystem>(FLAGS_bucket, fs);
 }
 
 void PrintError(const std::string& context_msg, const Status& st) {
@@ -90,13 +95,17 @@ void CheckDirectory(FileSystem* fs, const std::string& 
path) {
 }
 
 void ClearBucket(int argc, char** argv) {
-  auto fs = MakeFileSystem();
-
+  ASSERT_OK_AND_ASSIGN(auto fs, MakeFileSystem());
   ASSERT_OK(fs->DeleteRootDirContents());
 }
 
+void CreateBucket(int argc, char** argv) {
+  ASSERT_OK_AND_ASSIGN(auto fs, MakeRootFileSystem());
+  ASSERT_OK(fs->CreateDir(FLAGS_bucket));
+}
+
 void TestBucket(int argc, char** argv) {
-  auto fs = MakeFileSystem();
+  ASSERT_OK_AND_ASSIGN(auto fs, MakeFileSystem());
   std::vector<FileInfo> infos;
   FileSelector select;
   std::shared_ptr<io::InputStream> is;
@@ -221,13 +230,17 @@ void TestMain(int argc, char** argv) {
                           : (FLAGS_verbose ? S3LogLevel::Warn : 
S3LogLevel::Fatal);
   ASSERT_OK(InitializeS3(options));
 
-  if (FLAGS_region.empty()) {
+  if (FLAGS_region.empty() && FLAGS_endpoint.empty()) {
     ASSERT_OK_AND_ASSIGN(FLAGS_region, ResolveS3BucketRegion(FLAGS_bucket));
   }
 
+  if (FLAGS_create) {
+    CreateBucket(argc, argv);
+  }
   if (FLAGS_clear) {
     ClearBucket(argc, argv);
-  } else if (FLAGS_test) {
+  }
+  if (FLAGS_test) {
     TestBucket(argc, argv);
   }
 
@@ -244,8 +257,8 @@ int main(int argc, char** argv) {
   gflags::SetUsageMessage(ss.str());
   gflags::ParseCommandLineFlags(&argc, &argv, true);
 
-  if (FLAGS_clear + FLAGS_test != 1) {
-    ARROW_LOG(ERROR) << "Need exactly one of --test and --clear";
+  if (FLAGS_clear + FLAGS_test + FLAGS_create != 1) {
+    ARROW_LOG(ERROR) << "Need exactly one of --test, --clear and --create";
     return 2;
   }
   if (FLAGS_bucket.empty()) {