(asterixdb) branch master updated: [ASTERIXDB-3623][OTH] Wait for submitted cloud requests on exceeding max pending HTTP connections

[alsuliman]

This is an automated email from the ASF dual-hosted git repository.

alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git


The following commit(s) were added to refs/heads/master by this push:
     new ef7c1e1c36 [ASTERIXDB-3623][OTH] Wait for submitted cloud requests on 
exceeding max pending HTTP connections
ef7c1e1c36 is described below

commit ef7c1e1c3691d504c54063e8869a478161ff669b
Author: Ali Alsuliman <ali.al.solai...@gmail.com>
AuthorDate: Thu Jun 19 15:12:45 2025 -0700

    [ASTERIXDB-3623][OTH] Wait for submitted cloud requests on exceeding max 
pending HTTP connections
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Details:
    When downloading files in parallel, download up to max allowed pending,
    then wait for them to (acquire a connection and) finish.
    Continue with the rest.
    
    Ext-ref: MB-67243
    
    Change-Id: Ifd1a30c6eed5f316a2d7a17b685f537c24e6c0d2
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19954
    Reviewed-by: Ali Alsuliman <ali.al.solai...@gmail.com>
    Reviewed-by: Murtadha Hubail <mhub...@apache.org>
    Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
    Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
---
 .../cloud/clients/aws/s3/S3ParallelDownloader.java      | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git 
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
 
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
index 4d27c5a8c5..2acc12e7ba 100644
--- 
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
+++ 
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
@@ -77,8 +77,7 @@ class S3ParallelDownloader implements IParallelDownloader {
     @Override
     public void downloadFiles(Collection<FileReference> toDownload) throws 
HyracksDataException {
         try {
-            List<CompletableFuture<CompletedFileDownload>> downloads = 
startDownloadingFiles(toDownload);
-            waitForFileDownloads(downloads);
+            downloadFilesAndWait(toDownload);
         } catch (IOException | ExecutionException | InterruptedException e) {
             throw HyracksDataException.create(e);
         }
@@ -104,9 +103,10 @@ class S3ParallelDownloader implements IParallelDownloader {
         s3AsyncClient.close();
     }
 
-    private List<CompletableFuture<CompletedFileDownload>> 
startDownloadingFiles(Collection<FileReference> toDownload)
-            throws IOException {
+    private void downloadFilesAndWait(Collection<FileReference> toDownload)
+            throws IOException, ExecutionException, InterruptedException {
         List<CompletableFuture<CompletedFileDownload>> downloads = new 
ArrayList<>();
+        int maxPending = config.getRequestsMaxPendingHttpConnections();
         for (FileReference fileReference : toDownload) {
             // multipart download
             profiler.objectGet();
@@ -126,13 +126,18 @@ class S3ParallelDownloader implements IParallelDownloader 
{
 
             FileDownload fileDownload = 
transferManager.downloadFile(builder.build());
             downloads.add(fileDownload.completionFuture());
+            if (maxPending > 0 && downloads.size() >= maxPending) {
+                waitForFileDownloads(downloads);
+                downloads.clear();
+            }
+        }
+        if (!downloads.isEmpty()) {
+            waitForFileDownloads(downloads);
         }
-        return downloads;
     }
 
     private void 
waitForFileDownloads(List<CompletableFuture<CompletedFileDownload>> downloads)
             throws ExecutionException, InterruptedException {
-
         for (CompletableFuture<CompletedFileDownload> download : downloads) {
             download.get();
         }