>From Ali Alsuliman <[email protected]>:
Ali Alsuliman has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19954 )
Change subject: WIP: Wait for HTTP connections on exceeding max pending HTTP
connections
......................................................................
WIP: Wait for HTTP connections on exceeding max pending HTTP connections
Change-Id: Ifd1a30c6eed5f316a2d7a17b685f537c24e6c0d2
---
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
1 file changed, 15 insertions(+), 2 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/54/19954/1
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
index 4d27c5a..fc2d7f6 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
@@ -105,8 +105,9 @@
}
private List<CompletableFuture<CompletedFileDownload>>
startDownloadingFiles(Collection<FileReference> toDownload)
- throws IOException {
+ throws IOException, ExecutionException, InterruptedException {
List<CompletableFuture<CompletedFileDownload>> downloads = new
ArrayList<>();
+ int maxPending = config.getRequestsMaxPendingHttpConnections();
for (FileReference fileReference : toDownload) {
// multipart download
profiler.objectGet();
@@ -126,13 +127,16 @@
FileDownload fileDownload =
transferManager.downloadFile(builder.build());
downloads.add(fileDownload.completionFuture());
+ if (maxPending > 0 && downloads.size() >= maxPending) {
+ waitForFileDownloads(downloads);
+ downloads.clear();
+ }
}
return downloads;
}
private void
waitForFileDownloads(List<CompletableFuture<CompletedFileDownload>> downloads)
throws ExecutionException, InterruptedException {
-
for (CompletableFuture<CompletedFileDownload> download : downloads) {
download.get();
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19954
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ifd1a30c6eed5f316a2d7a17b685f537c24e6c0d2
Gerrit-Change-Number: 19954
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <[email protected]>
Gerrit-MessageType: newchange
