Author: mreutegg
Date: Thu Jan 11 13:32:01 2018
New Revision: 1820877
URL: http://svn.apache.org/viewvc?rev=1820877&view=rev
Log:
OAK-7140: Retry query on MongoException
Modified:
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/RetryReadIT.java
Modified:
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java?rev=1820877&r1=1820876&r2=1820877&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
Thu Jan 11 13:32:01 2018
@@ -220,6 +220,13 @@ public class MongoDocumentStore implemen
private int bulkRetries =
Integer.getInteger("oak.mongo.bulkRetries", 0);
+ /**
+ * How many times a query to MongoDB should be retried when it fails with a
+ * MongoException.
+ */
+ private final int queryRetries =
+ Integer.getInteger("oak.mongo.queryRetries", 2);
+
private String lastReadWriteMode;
private final Map<String, String> metadata;
@@ -429,7 +436,7 @@ public class MongoDocumentStore implemen
final int maxCacheAge) {
if (collection != Collection.NODES) {
return findUncachedWithRetry(collection, key,
- DocumentReadPreference.PRIMARY, 2);
+ DocumentReadPreference.PRIMARY);
}
NodeDocument doc;
if (maxCacheAge > 0 || preferCached) {
@@ -467,7 +474,7 @@ public class MongoDocumentStore implemen
}
final NodeDocument d = (NodeDocument) findUncachedWithRetry(
collection, key,
- getReadPreference(maxCacheAge), 2);
+ getReadPreference(maxCacheAge));
invalidateCache(collection, key);
doc = nodesCache.get(key, new Callable<NodeDocument>() {
@Override
@@ -500,20 +507,17 @@ public class MongoDocumentStore implemen
* @param collection the collection to read from.
* @param key the key of the document to find.
* @param docReadPref the read preference.
- * @param retries the number of retries. Must not be negative.
* @param <T> the document type of the given collection.
* @return the document or {@code null} if the document doesn't exist.
*/
@CheckForNull
private <T extends Document> T findUncachedWithRetry(
Collection<T> collection, String key,
- DocumentReadPreference docReadPref,
- int retries) {
- checkArgument(retries >= 0, "retries must not be negative");
+ DocumentReadPreference docReadPref) {
if (key.equals("0:/")) {
LOG.trace("root node");
}
- int numAttempts = retries + 1;
+ int numAttempts = queryRetries + 1;
MongoException ex = null;
for (int i = 0; i < numAttempts; i++) {
if (i > 0) {
@@ -581,11 +585,40 @@ public class MongoDocumentStore implemen
String indexedProperty,
long startValue,
int limit) {
- try {
- return queryInternal(collection, fromKey, toKey, indexedProperty,
- startValue, limit, maxQueryTimeMS);
- } catch (MongoException e) {
- throw handleException(e, collection, Lists.newArrayList(fromKey,
toKey));
+ return queryWithRetry(collection, fromKey, toKey, indexedProperty,
+ startValue, limit, maxQueryTimeMS);
+ }
+
+ /**
+ * Queries for documents and performs a number of retries if the read fails
+ * with an exception.
+ */
+ @Nonnull
+ private <T extends Document> List<T> queryWithRetry(Collection<T>
collection,
+ String fromKey,
+ String toKey,
+ String indexedProperty,
+ long startValue,
+ int limit,
+ long maxQueryTime) {
+ int numAttempts = queryRetries + 1;
+ MongoException ex = null;
+ for (int i = 0; i < numAttempts; i++) {
+ if (i > 0) {
+ LOG.warn("Retrying query, fromKey={}, toKey={}", fromKey,
toKey);
+ }
+ try {
+ return queryInternal(collection, fromKey, toKey,
+ indexedProperty, startValue, limit, maxQueryTime);
+ } catch (MongoException e) {
+ ex = e;
+ }
+ }
+ if (ex != null) {
+ throw handleException(ex, collection, Lists.newArrayList(fromKey,
toKey));
+ } else {
+ // impossible to get here
+ throw new IllegalStateException();
}
}
Modified:
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/RetryReadIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/RetryReadIT.java?rev=1820877&r1=1820876&r2=1820877&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/RetryReadIT.java
(original)
+++
jackrabbit/oak/trunk/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/RetryReadIT.java
Thu Jan 11 13:32:01 2018
@@ -16,6 +16,10 @@
*/
package org.apache.jackrabbit.oak.plugins.document.mongo;
+import java.util.List;
+
+import javax.annotation.Nonnull;
+
import org.apache.jackrabbit.oak.plugins.document.AbstractMongoConnectionTest;
import org.apache.jackrabbit.oak.plugins.document.Collection;
import org.apache.jackrabbit.oak.plugins.document.Document;
@@ -30,7 +34,10 @@ import com.mongodb.DB;
import com.mongodb.MongoException;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
+import static org.hamcrest.collection.IsEmptyCollection.empty;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
/**
@@ -69,6 +76,29 @@ public class RetryReadIT extends Abstrac
}
}
+ @Test
+ public void retryQuery() {
+ String fromKey = Utils.getKeyLowerLimit("/foo");
+ String toKey = Utils.getKeyUpperLimit("/foo");
+ // must survive two consecutive failures. -> 2 retries
+ store.failRead = 2;
+ List<NodeDocument> docs = store.query(NODES, fromKey, toKey, 100);
+ assertThat(docs, is(empty()));
+
+ fromKey = Utils.getKeyLowerLimit("/bar");
+ toKey = Utils.getKeyUpperLimit("/bar");
+ // must fail with three consecutive failures
+ store.failRead = 3;
+ try {
+ store.query(NODES, fromKey, toKey, 100);
+ fail("must fail with DocumentStoreException");
+ } catch (DocumentStoreException e) {
+ // expected
+ } finally {
+ store.failRead = 0;
+ }
+ }
+
private static class TestStore extends MongoDocumentStore {
private int failRead = 0;
@@ -81,12 +111,29 @@ public class RetryReadIT extends Abstrac
protected <T extends Document> T findUncached(Collection<T> collection,
String key,
DocumentReadPreference
docReadPref) {
+ maybeFail();
+ return super.findUncached(collection, key, docReadPref);
+ }
+
+ @Nonnull
+ @Override
+ protected <T extends Document> List<T> queryInternal(Collection<T>
collection,
+ String fromKey,
+ String toKey,
+ String
indexedProperty,
+ long startValue,
+ int limit,
+ long
maxQueryTime) {
+ maybeFail();
+ return super.queryInternal(collection, fromKey, toKey,
+ indexedProperty, startValue, limit, maxQueryTime);
+ }
+
+ private void maybeFail() {
if (failRead > 0) {
failRead--;
throw new MongoException("read failed");
}
- return super.findUncached(collection, key, docReadPref);
}
}
-
}
