pitrou commented on a change in pull request #8101:
URL: https://github.com/apache/arrow/pull/8101#discussion_r482921231
##########
File path: cpp/src/arrow/filesystem/filesystem.cc
##########
@@ -439,6 +440,28 @@ Result<std::shared_ptr<io::OutputStream>>
SlowFileSystem::OpenAppendStream(
return base_fs_->OpenAppendStream(path);
}
+Status CopyFiles(const std::shared_ptr<FileSystem>& src_fs,
+ const std::vector<std::string>& src_paths,
+ const std::shared_ptr<FileSystem>& dest_fs,
+ const std::vector<std::string>& dest_paths, int64_t
chunk_size,
+ bool use_threads) {
+ if (src_paths.size() != dest_paths.size()) {
+ return Status::Invalid("Trying to copy ", src_paths.size(), " files into ",
+ dest_paths.size(), " paths.");
+ }
+
+ return ::arrow::internal::OptionalParallelFor(
+ use_threads, static_cast<int>(src_paths.size()), [&](int i) {
+ ARROW_ASSIGN_OR_RAISE(auto src, src_fs->OpenInputStream(src_paths[i]));
+ auto dest_dir = internal::GetAbstractPathParent(dest_paths[i]).first;
+ if (!dest_dir.empty()) {
+ RETURN_NOT_OK(dest_fs->CreateDir(dest_dir));
+ }
+ ARROW_ASSIGN_OR_RAISE(auto dest,
dest_fs->OpenOutputStream(dest_paths[i]));
+ return internal::CopyStream(src, dest, chunk_size);
Review comment:
Writing is typically asynchronous. A separate write thread may speed up
things a little, but I don't think it would make a sizable difference.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
