Smalyshev has uploaded a new change for review.
https://gerrit.wikimedia.org/r/208826
Change subject: T96713: add retries in HTTP calls
......................................................................
T96713: add retries in HTTP calls
Change-Id: I0c681559d0d3c9d93250dd80e2be67d1b7e318d2
---
M tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
1 file changed, 38 insertions(+), 9 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikidata/query/rdf
refs/changes/26/208826/1
diff --git
a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
index aa408d7..28beafb 100644
--- a/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
+++ b/tools/src/main/java/org/wikidata/query/rdf/tool/rdf/RdfRepository.java
@@ -106,6 +106,17 @@
private final String updateLeftOffTimeBody;
/**
+ * How many times we retry a failed HTTP call.
+ * FIXME: make configurable
+ */
+ private final int maxRetries = 5;
+ /**
+ * How long to delay after failing HTTP call, in milliseconds.
+ * FIXME: make configurable
+ */
+ private final int delay = 1000;
+
+ /**
* Allow subclass access to the HTTP client.
*/
protected CloseableHttpClient client() {
@@ -331,17 +342,35 @@
List<NameValuePair> entity = new ArrayList<>();
entity.add(new BasicNameValuePair(type, sparql));
post.setEntity(new UrlEncodedFormEntity(entity, Consts.UTF_8));
- try {
- try (CloseableHttpResponse response = client.execute(post)) {
- if (response.getStatusLine().getStatusCode() != 200) {
- throw new ContainedException("Non-200 response from triple
store: " + response + " body=\n"
- + responseBodyAsString(response));
+ int retries = 0;
+ do {
+ try {
+ try (CloseableHttpResponse response = client.execute(post)) {
+ if (response.getStatusLine().getStatusCode() != 200) {
+ throw new ContainedException("Non-200 response from
triple store: " + response + " body=\n"
+ + responseBodyAsString(response));
+ }
+ return responseHandler.parse(response.getEntity());
}
- return responseHandler.parse(response.getEntity());
+ } catch (IOException e) {
+ if (retries < maxRetries) {
+ // Increasing delay, with random 10% variation so threads
won't all get restarts
+ // at the same time.
+ int retryIn = (int)Math.ceil(delay * (retries + 1) * (1 +
Math.random() * 0.1));
+ log.info("HTTP request failed: {}, retrying in {} ms", e,
retryIn);
+ retries++;
+ try {
+ Thread.sleep(retryIn);
+ } catch (InterruptedException e1) {
+ // Don't care if interrupted
+ }
+ continue;
+ }
+ // FIXME: if all retries failed, what should we do?
+ throw new RuntimeException("Error updating triple store", e);
}
- } catch (IOException e) {
- throw new RuntimeException("Error updating triple store", e);
- }
+ } while (true);
+
}
/**
--
To view, visit https://gerrit.wikimedia.org/r/208826
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I0c681559d0d3c9d93250dd80e2be67d1b7e318d2
Gerrit-PatchSet: 1
Gerrit-Project: wikidata/query/rdf
Gerrit-Branch: master
Gerrit-Owner: Smalyshev <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
