This is an automated email from the ASF dual-hosted git repository. alsuliman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5f2c24f4db895fe6d91f90336ab2502783b2a8f7 Author: Hussain Towaileb <[email protected]> AuthorDate: Mon Mar 6 21:41:16 2023 +0300 [NO ISSUE][EXT]: Properly handle failures and retries for GCS Change-Id: I0bd42c391db603040d0470e976fa561649024992 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17409 Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> --- .../input/record/reader/gcs/GCSInputStream.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java index 652fa3e4b0..5da4583935 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java @@ -47,7 +47,7 @@ public class GCSInputStream extends AbstractExternalInputStream { private final Storage client; private final String container; - private static final int MAX_RETRIES = 5; // We will retry 5 times in case of retryable errors + private static final int MAX_ATTEMPTS = 5; // We try a total of 5 times in case of retryable errors public GCSInputStream(Map<String, String> configuration, List<String> filePaths) throws HyracksDataException { super(configuration, filePaths); @@ -78,10 +78,10 @@ public class GCSInputStream extends AbstractExternalInputStream { * @return true */ private boolean doGetInputStream(String fileName) throws RuntimeDataException { - int retries = 0; + int attempt = 0; BlobId blobId = BlobId.of(container, fileName); - while (retries < MAX_RETRIES) { + while (attempt < MAX_ATTEMPTS) { try { Blob blob = client.get(blobId); if (blob == null) { @@ -93,14 +93,14 @@ public class GCSInputStream extends AbstractExternalInputStream { in = new ByteArrayInputStream(blob.getContent()); break; } catch (BaseServiceException ex) { - if (!shouldRetry(retries++) && ex.isRetryable()) { + if (!ex.isRetryable() || !shouldRetry(++attempt)) { throw new RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex)); } - LOGGER.debug(() -> "Retryable error: " + LogRedactionUtil.userData(ex.getMessage())); + LOGGER.debug(() -> "Retryable error: " + getMessageOrToString(ex)); - // Backoff for 1 sec for the first 2 retries, and 2 seconds from there onward + // Backoff for 1 sec for the first 3 attempts, and 2 seconds from there onward try { - Thread.sleep(TimeUnit.SECONDS.toMillis(retries < 3 ? 1 : 2)); + Thread.sleep(TimeUnit.SECONDS.toMillis(attempt < 3 ? 1 : 2)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } @@ -111,8 +111,8 @@ public class GCSInputStream extends AbstractExternalInputStream { return true; } - private boolean shouldRetry(int currentRetry) { - return currentRetry < MAX_RETRIES; + private boolean shouldRetry(int nextAttempt) { + return nextAttempt < MAX_ATTEMPTS; } @Override
