This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 8fed97fa95 GH-33904: [R] improve behavior of s3_bucket - work-around
(#34009)
8fed97fa95 is described below
commit 8fed97fa951032a7e686407d29107cf48f6ab5f5
Author: Carl Boettiger <[email protected]>
AuthorDate: Fri Feb 10 10:37:26 2023 -0800
GH-33904: [R] improve behavior of s3_bucket - work-around (#34009)
I think this is another option to address #33918.
This retains the existing behavior when no additional arguments (region,
endpoint, etc) are supplied.
If any optional arguments are supplied, this approach will by-pass the
`$from_uri` call. I think this is reasonable, given that the `from_uri()` call
is implicitly ignoring optional arguments already, which is giving rise to the
current bugs.
* Closes: #33904
Lead-authored-by: Carl Boettiger <[email protected]>
Co-authored-by: Carl Boettiger <[email protected]>
Co-authored-by: Carl <[email protected]>
Co-authored-by: Dewey Dunnington <[email protected]>
Signed-off-by: Dewey Dunnington <[email protected]>
---
r/R/filesystem.R | 24 +++++++++++++-----------
r/tests/testthat/test-s3-minio.R | 18 ++++++++++++++++++
2 files changed, 31 insertions(+), 11 deletions(-)
diff --git a/r/R/filesystem.R b/r/R/filesystem.R
index 74760ad1a7..dc95fda91f 100644
--- a/r/R/filesystem.R
+++ b/r/R/filesystem.R
@@ -458,20 +458,22 @@ s3_bucket <- function(bucket, ...) {
assert_that(is.string(bucket))
args <- list2(...)
- # Use FileSystemFromUri to detect the bucket's region
- if (!is_url(bucket)) {
- bucket <- paste0("s3://", bucket)
- }
- fs_and_path <- FileSystem$from_uri(bucket)
- fs <- fs_and_path$fs
- # If there are no additional S3Options, we can use that filesystem
- # Otherwise, take the region that was detected and make a new fs with the
args
- if (length(args)) {
- args$region <- fs$region
+ # If user specifies args, they must specify region as arg, env var, or config
+ if (length(args) == 0) {
+ # Use FileSystemFromUri to detect the bucket's region
+ if (!is_url(bucket)) {
+ bucket <- paste0("s3://", bucket)
+ }
+
+ fs_and_path <- FileSystem$from_uri(bucket)
+ fs <- fs_and_path$fs
+ } else {
+ # If there are no additional S3Options, we can use that filesystem
fs <- exec(S3FileSystem$create, !!!args)
}
+
# Return a subtree pointing at that bucket path
- SubTreeFileSystem$create(fs_and_path$path, fs)
+ SubTreeFileSystem$create(bucket, fs)
}
#' Connect to a Google Cloud Storage (GCS) bucket
diff --git a/r/tests/testthat/test-s3-minio.R b/r/tests/testthat/test-s3-minio.R
index 2e72cc1501..8dfac63471 100644
--- a/r/tests/testthat/test-s3-minio.R
+++ b/r/tests/testthat/test-s3-minio.R
@@ -106,5 +106,23 @@ test_that("S3FileSystem input validation", {
)
})
+test_that("Confirm s3_bucket works with endpoint_override", {
+ bucket <- s3_bucket(
+ now,
+ access_key = minio_key,
+ secret_key = minio_secret,
+ scheme = "http",
+ endpoint_override = paste0("localhost:", minio_port)
+ )
+
+ expect_r6_class(bucket, "SubTreeFileSystem")
+
+ os <- bucket$OpenOutputStream("bucket-test.csv")
+ write_csv_arrow(example_data, os)
+ os$close()
+ expect_true("bucket-test.csv" %in% bucket$ls())
+ bucket$DeleteFile("bucket-test.csv")
+})
+
# Cleanup
withr::deferred_run()
