(hadoop) branch branch-3.4 updated: HADOOP-19210. S3A: Speed up some slow unit tests (#6907)

[stevel]

This is an automated email from the ASF dual-hosted git repository.

stevel pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.4 by this push:
     new b5e21f94950 HADOOP-19210. S3A: Speed up some slow unit tests (#6907)
b5e21f94950 is described below

commit b5e21f949508f6837502c6613d8a914b59cf6cde
Author: Steve Loughran <ste...@cloudera.com>
AuthorDate: Tue Jul 2 11:34:45 2024 +0100

    HADOOP-19210. S3A: Speed up some slow unit tests (#6907)
    
    Speed up slow tests
    * TestS3AAWSCredentialsProvider: decrease thread pool shutdown time
    * TestS3AInputStreamRetry: reduce retry limit and intervals
    
    Contributed by Steve Loughran
---
 .../test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java  | 9 +++++++++
 .../org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java  | 8 +++++---
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
index f43710cf25e..e76b3046048 100644
--- 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
+++ 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
@@ -80,6 +80,15 @@ public abstract class AbstractS3AMockTest {
     conf.setInt(ASYNC_DRAIN_THRESHOLD, Integer.MAX_VALUE);
     // set the region to avoid the getBucketLocation on FS init.
     conf.set(AWS_REGION, "eu-west-1");
+
+    // tight retry logic as all failures are simulated
+    final String interval = "1ms";
+    final int limit = 3;
+    conf.set(RETRY_THROTTLE_INTERVAL, interval);
+    conf.setInt(RETRY_THROTTLE_LIMIT, limit);
+    conf.set(RETRY_INTERVAL, interval);
+    conf.setInt(RETRY_LIMIT, limit);
+
     return conf;
   }
 
diff --git 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
index 0ffd7e75b18..d51bc954a63 100644
--- 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
@@ -86,6 +86,8 @@ public class TestS3AAWSCredentialsProvider extends 
AbstractS3ATestBase {
 
   private static final Logger LOG = 
LoggerFactory.getLogger(TestS3AAWSCredentialsProvider.class);
 
+  public static final int TERMINATION_TIMEOUT = 3;
+
   @Test
   public void testProviderWrongClass() throws Exception {
     expectProviderInstantiationFailure(this.getClass(),
@@ -579,7 +581,7 @@ public class TestS3AAWSCredentialsProvider extends 
AbstractS3ATestBase {
     }
   }
 
-  private static final int CONCURRENT_THREADS = 10;
+  private static final int CONCURRENT_THREADS = 4;
 
   @Test
   public void testConcurrentAuthentication() throws Throwable {
@@ -619,7 +621,7 @@ public class TestS3AAWSCredentialsProvider extends 
AbstractS3ATestBase {
             "expectedSecret", credentials.secretAccessKey());
       }
     } finally {
-      pool.awaitTermination(10, TimeUnit.SECONDS);
+      pool.awaitTermination(TERMINATION_TIMEOUT, TimeUnit.SECONDS);
       pool.shutdown();
     }
 
@@ -685,7 +687,7 @@ public class TestS3AAWSCredentialsProvider extends 
AbstractS3ATestBase {
         );
       }
     } finally {
-      pool.awaitTermination(10, TimeUnit.SECONDS);
+      pool.awaitTermination(TERMINATION_TIMEOUT, TimeUnit.SECONDS);
       pool.shutdown();
     }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org