pitrou commented on a change in pull request #10958:
URL: https://github.com/apache/arrow/pull/10958#discussion_r712261963
##########
File path: python/pyarrow/tests/test_fs.py
##########
@@ -1547,3 +1549,107 @@ def test_s3_real_aws_region_selection():
fs, path = FileSystem.from_uri(
's3://x-arrow-non-existent-bucket?region=us-east-3')
assert fs.region == 'us-east-3'
+
+
[email protected]
+def test_copy_files(s3_connection, s3fs, tempdir):
+ fs = s3fs["fs"]
+ pathfn = s3fs["pathfn"]
+
+ # create test file on S3 filesystem
+ c = pathfn('c.txt')
+ with fs.open_output_stream(c) as f:
+ f.write(b'test')
+
+ # create URI for created file
+ host, port, access_key, secret_key = s3_connection
+ source_uri = "s3://{}:{}@{}?scheme=http&endpoint_override={}:{}" \
+ .format(access_key, secret_key, c, host, port)
Review comment:
Can use a f-string here.
##########
File path: python/pyarrow/fs.py
##########
@@ -178,10 +182,74 @@ def _resolve_filesystem_and_path(
# instead of a more confusing scheme parsing error
if "empty scheme" not in str(e):
raise
+ else:
+ path = filesystem.normalize_path(path)
return filesystem, path
+def copy_files(source, destination,
+ source_filesystem=None, destination_filesystem=None,
+ chunk_size=1024*1024, use_threads=True):
Review comment:
`chunk_size` and `use_threads` should probably be keyword-only?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
