>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17409 )
Change subject: [NO ISSUE][EXT]: Properly handle failures and retries for GCS
......................................................................
[NO ISSUE][EXT]: Properly handle failures and retries for GCS
Change-Id: I0bd42c391db603040d0470e976fa561649024992
---
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
1 file changed, 18 insertions(+), 9 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/09/17409/1
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
index 652fa3e..5da4583 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/gcs/GCSInputStream.java
@@ -47,7 +47,7 @@
private final Storage client;
private final String container;
- private static final int MAX_RETRIES = 5; // We will retry 5 times in case
of retryable errors
+ private static final int MAX_ATTEMPTS = 5; // We try a total of 5 times in
case of retryable errors
public GCSInputStream(Map<String, String> configuration, List<String>
filePaths) throws HyracksDataException {
super(configuration, filePaths);
@@ -78,10 +78,10 @@
* @return true
*/
private boolean doGetInputStream(String fileName) throws
RuntimeDataException {
- int retries = 0;
+ int attempt = 0;
BlobId blobId = BlobId.of(container, fileName);
- while (retries < MAX_RETRIES) {
+ while (attempt < MAX_ATTEMPTS) {
try {
Blob blob = client.get(blobId);
if (blob == null) {
@@ -93,14 +93,14 @@
in = new ByteArrayInputStream(blob.getContent());
break;
} catch (BaseServiceException ex) {
- if (!shouldRetry(retries++) && ex.isRetryable()) {
+ if (!ex.isRetryable() || !shouldRetry(++attempt)) {
throw new
RuntimeDataException(ErrorCode.EXTERNAL_SOURCE_ERROR, getMessageOrToString(ex));
}
- LOGGER.debug(() -> "Retryable error: " +
LogRedactionUtil.userData(ex.getMessage()));
+ LOGGER.debug(() -> "Retryable error: " +
getMessageOrToString(ex));
- // Backoff for 1 sec for the first 2 retries, and 2 seconds
from there onward
+ // Backoff for 1 sec for the first 3 attempts, and 2 seconds
from there onward
try {
- Thread.sleep(TimeUnit.SECONDS.toMillis(retries < 3 ? 1 :
2));
+ Thread.sleep(TimeUnit.SECONDS.toMillis(attempt < 3 ? 1 :
2));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@@ -111,8 +111,8 @@
return true;
}
- private boolean shouldRetry(int currentRetry) {
- return currentRetry < MAX_RETRIES;
+ private boolean shouldRetry(int nextAttempt) {
+ return nextAttempt < MAX_ATTEMPTS;
}
@Override
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17409
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: neo
Gerrit-Change-Id: I0bd42c391db603040d0470e976fa561649024992
Gerrit-Change-Number: 17409
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
