hutiefang76 commented on code in PR #27187: URL: https://github.com/apache/flink/pull/27187#discussion_r3995859775
########## flink-filesystems/flink-s3-fs-native/src/main/java/org/apache/flink/fs/s3native/NativeS3BulkCopyHelper.java: ########## @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.fs.s3native; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.core.fs.ICloseableRegistry; +import org.apache.flink.core.fs.PathsCopyingFileSystem; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.transfer.s3.S3TransferManager; +import software.amazon.awssdk.transfer.s3.model.CompletedCopy; +import software.amazon.awssdk.transfer.s3.model.DownloadFileRequest; +import software.amazon.awssdk.transfer.s3.model.FileDownload; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +@Internal +public class NativeS3BulkCopyHelper { + + private static final Logger LOG = LoggerFactory.getLogger(NativeS3BulkCopyHelper.class); + + private final S3TransferManager transferManager; + private final int maxConcurrentCopies; + + public NativeS3BulkCopyHelper(S3TransferManager transferManager, int maxConcurrentCopies) { + this.transferManager = transferManager; + this.maxConcurrentCopies = maxConcurrentCopies; + } + + public void copyFiles( + List<PathsCopyingFileSystem.CopyRequest> requests, ICloseableRegistry closeableRegistry) + throws IOException { + + if (requests.isEmpty()) { + return; + } + + LOG.info("Starting bulk copy of {} files using S3TransferManager", requests.size()); + + List<CompletableFuture<CompletedCopy>> copyFutures = new ArrayList<>(); + + for (int i = 0; i < requests.size(); i++) { + PathsCopyingFileSystem.CopyRequest request = requests.get(i); + String sourceUri = request.getSource().toUri().toString(); + if (sourceUri.startsWith("s3://") || sourceUri.startsWith("s3a://")) { + copyS3ToLocal(request, copyFutures); + } else { + throw new UnsupportedOperationException( + "Only S3 to local copies are currently supported: " + sourceUri); + } + + if (copyFutures.size() >= maxConcurrentCopies || i == requests.size() - 1) { Review Comment: @Izeren I would like to take FLINK-39116, following up on this review comment. Is anyone already working on it? Current master already has a fixed download pool, but still waits for each whole batch. I prepared a prototype that uses a completion queue to replenish the existing bounded window. It starts getObject only when a slot is available and releases the slot after response cleanup. The concurrency/connection settings and IOException cancellation contract stay unchanged. The regression tests hold the first request pending while the second finishes and the third starts; the original code fails both cases (normal concurrency and connection-pool clamping). The patch also checks failure after replenishment and includes a SeaweedFS bulk-copy integration test. Prototype: https://github.com/hutiefang76/flink/commit/088a590e7660d58c9abd3c64218563c97e9eeb95 Targeted native S3 reactor clean verify passes on Java 17, with 303 selected tests passing in both Surefire phases. Would this approach be appropriate for the ticket? I will follow the Jira assignment process before opening a PR. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
