This is an automated email from the ASF dual-hosted git repository. stefanegli pushed a commit to branch OAK-10281-4 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 3e94335e56af8ba5e395e66cccaa97bc559da737 Author: Stefan Egli <[email protected]> AuthorDate: Thu Feb 8 12:35:27 2024 +0100 OAK-10281 : draft of handling recoveryDelayMillis directly as parameter in isRecoveryNeeded --- .../plugins/document/RevisionContextWrapper.java | 5 +++ .../apache/jackrabbit/oak/run/RecoveryCommand.java | 4 +-- .../oak/plugins/document/ClusterNodeInfo.java | 42 ++++++++-------------- .../plugins/document/ClusterNodeInfoDocument.java | 6 ++-- .../oak/plugins/document/DocumentNodeStore.java | 14 ++++++-- .../plugins/document/DocumentNodeStoreBuilder.java | 2 +- .../plugins/document/DocumentNodeStoreService.java | 3 -- .../oak/plugins/document/LastRevRecoveryAgent.java | 15 ++++---- .../oak/plugins/document/MissingLastRevSeeker.java | 13 ++++--- .../oak/plugins/document/RecoveryContext.java | 8 +++++ .../oak/plugins/document/RecoveryHandlerImpl.java | 5 ++- .../oak/plugins/document/RecoveryLock.java | 7 ++-- .../oak/plugins/document/RevisionContext.java | 5 +++ .../mongo/MongoDocumentNodeStoreBuilderBase.java | 2 +- .../document/mongo/MongoMissingLastRevSeeker.java | 4 +-- .../document/rdb/RDBDocumentNodeStoreBuilder.java | 2 +- .../document/rdb/RDBMissingLastRevSeeker.java | 4 +-- .../oak/plugins/document/ClusterNodeInfoTest.java | 27 +++++++------- .../document/DocumentNodeStoreServiceTest.java | 3 +- .../plugins/document/DocumentNodeStoreTest.java | 3 +- .../oak/plugins/document/DocumentSplitTest.java | 5 +++ .../oak/plugins/document/DummyRevisionContext.java | 5 +++ .../oak/plugins/document/LastRevRecoveryTest.java | 16 +++++---- .../plugins/document/MissingLastRevSeekerTest.java | 24 +++++++------ .../plugins/document/NodeDocumentSweeperIT.java | 11 +++--- .../oak/plugins/document/RecoveryContextTest.java | 14 ++++++++ .../oak/plugins/document/RecoveryHandlerTest.java | 5 +-- .../oak/plugins/document/RecoveryLockTest.java | 16 +++++---- .../plugins/document/SimpleRecoveryHandler.java | 8 +++-- .../document/mongo/AcquireRecoveryLockTest.java | 4 ++- .../document/mongo/LeaseUpdateSocketTimeoutIT.java | 5 ++- .../plugins/document/rdb/RDBDocumentStoreTest.java | 5 ++- 32 files changed, 181 insertions(+), 111 deletions(-) diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/RevisionContextWrapper.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/RevisionContextWrapper.java index 4d5b7e7b52..0886939fbf 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/RevisionContextWrapper.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/document/RevisionContextWrapper.java @@ -67,6 +67,11 @@ public final class RevisionContextWrapper implements RevisionContext { return context.getClock(); } + @Override + public long getRecoveryDelayMillis() { + return context.getRecoveryDelayMillis(); + } + @Override public String getCommitValue(@NotNull Revision revision, @NotNull NodeDocument nodeDocument) { diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RecoveryCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RecoveryCommand.java index b2f7b5d1b6..4bee9d45cc 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RecoveryCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RecoveryCommand.java @@ -86,11 +86,11 @@ class RecoveryCommand implements Command { if (ds instanceof MongoDocumentStore) { MongoDocumentStore docStore = (MongoDocumentStore) ds; agent = new LastRevRecoveryAgent(docStore, dns); - seeker = new MongoMissingLastRevSeeker(docStore, dns.getClock()); + seeker = new MongoMissingLastRevSeeker(docStore, dns.getClock(), dns.getRecoveryDelayMillis()); } else if (ds instanceof RDBDocumentStore) { RDBDocumentStore docStore = (RDBDocumentStore) ds; agent = new LastRevRecoveryAgent(docStore, dns); - seeker = new RDBMissingLastRevSeeker(docStore, dns.getClock()); + seeker = new RDBMissingLastRevSeeker(docStore, dns.getClock(), dns.getRecoveryDelayMillis()); } if (agent == null || seeker == null) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java index fe83374122..adff46ed04 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfo.java @@ -238,14 +238,7 @@ public class ClusterNodeInfo { static final int DEFAULT_REUSE_DELAY_AFTER_RECOVERY_MILLIS = 0; /** OAK-10281 : default millis to delay a recovery after a lease timeout */ - static final long DEFAULT_RECOVERY_DELAY_MILLIS = 0; - - /** - * Actual millis to delay a recovery after a lease timeout. - * <p> - * Initialized by DocumentNodeStore constructor. - */ - static long recoveryDelayMillis = DEFAULT_RECOVERY_DELAY_MILLIS; + public static final long DEFAULT_RECOVERY_DELAY_MILLIS = 0; /** * The Oak version. @@ -476,7 +469,8 @@ public class ClusterNodeInfo { int configuredClusterId, boolean invisible) { return getInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId, - invisible, DEFAULT_REUSE_DELAY_AFTER_RECOVERY_MILLIS); + invisible, DEFAULT_REUSE_DELAY_AFTER_RECOVERY_MILLIS, + DEFAULT_RECOVERY_DELAY_MILLIS); } /** @@ -496,7 +490,8 @@ public class ClusterNodeInfo { String instanceId, int configuredClusterId, boolean invisible, - long reuseAfterRecoveryMillis) { + long reuseAfterRecoveryMillis, + long recoveryDelayMillis) { // defaults for machineId and instanceID if (machineId == null) { machineId = MACHINE_ID; @@ -509,7 +504,8 @@ public class ClusterNodeInfo { for (int i = 0; i < retries; i++) { Map.Entry<ClusterNodeInfo, Long> suggestedClusterNode = createInstance(store, recoveryHandler, machineId, - instanceId, configuredClusterId, i == 0, invisible, reuseAfterRecoveryMillis); + instanceId, configuredClusterId, i == 0, invisible, + reuseAfterRecoveryMillis, recoveryDelayMillis); ClusterNodeInfo clusterNode = suggestedClusterNode.getKey(); Long currentStartTime = suggestedClusterNode.getValue(); String key = String.valueOf(clusterNode.id); @@ -565,7 +561,8 @@ public class ClusterNodeInfo { int configuredClusterId, boolean waitForLease, boolean invisible, - long reuseAfterRecoveryMillis) { + long reuseAfterRecoveryMillis, + long recoveryDelayMillis) { long now = getCurrentTime(); int maxId = 0; @@ -616,7 +613,7 @@ public class ClusterNodeInfo { // -> potentially wait for lease if machine and instance id match if (leaseEnd != null && leaseEnd > now - && !doc.isRecoveryNeeded(now)) { + && !doc.isRecoveryNeeded(now, recoveryDelayMillis)) { // wait if (a) instructed to, and (b) also the remaining time // time is not much bigger than the lease interval (in which // case something is very very wrong anyway) @@ -624,7 +621,9 @@ public class ClusterNodeInfo { && iId.equals(instanceId)) { boolean worthRetrying = waitForLeaseExpiry(store, doc, leaseEnd, machineId, instanceId); if (worthRetrying) { - return createInstance(store, recoveryHandler, machineId, instanceId, configuredClusterId, false, invisible, reuseAfterRecoveryMillis); + return createInstance(store, recoveryHandler, machineId, + instanceId, configuredClusterId, false, invisible, + reuseAfterRecoveryMillis, recoveryDelayMillis); } } @@ -636,7 +635,7 @@ public class ClusterNodeInfo { // if we get here the clusterId either: // 1) is inactive // 2) needs recovery - if (doc.isRecoveryNeeded(now)) { + if (doc.isRecoveryNeeded(now, recoveryDelayMillis)) { if (mId.equals(machineId) && iId.equals(instanceId)) { // this id matches our environment and has an expired lease // use it after a successful recovery @@ -1235,19 +1234,6 @@ public class ClusterNodeInfo { clock = Clock.SIMPLE; } - static long getRecoveryDelayMillis() { - return recoveryDelayMillis; - } - - static void setRecoveryDelayMillis(long recoveryDelayMillis) { - ClusterNodeInfo.recoveryDelayMillis = recoveryDelayMillis; - } - - /** <b>only used for testing</b> **/ - static void resetRecoveryDelayMillisToDefault() { - recoveryDelayMillis = DEFAULT_RECOVERY_DELAY_MILLIS; - } - private static long getProcessId() { try { String name = ManagementFactory.getRuntimeMXBean().getName(); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java index adfb83161d..1e0ad8974a 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoDocument.java @@ -105,10 +105,12 @@ public class ClusterNodeInfoDocument extends Document { * </ul> * @param currentTimeMillis the current time in milliseconds since the start * start of the epoch. + * @param recoveryDelayMillis the configured time in millis which a recovery + * should be delayed by. */ - public boolean isRecoveryNeeded(long currentTimeMillis) { + public boolean isRecoveryNeeded(long currentTimeMillis, long recoveryDelayMillis) { return isActive() && - (currentTimeMillis - getLeaseEndTime() > ClusterNodeInfo.getRecoveryDelayMillis() || + (currentTimeMillis - getLeaseEndTime() > recoveryDelayMillis || isBeingRecovered()); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java index 33c34db26e..c99cd73210 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java @@ -505,6 +505,8 @@ public final class DocumentNodeStore private final Clock clock; + private final long recoveryDelayMillis; + private final Checkpoints checkpoints; private final VersionGarbageCollector versionGarbageCollector; @@ -598,6 +600,7 @@ public final class DocumentNodeStore this.executor = builder.getExecutor(); this.lastRevSeeker = builder.createMissingLastRevSeeker(); this.clock = builder.getClock(); + this.recoveryDelayMillis = builder.getRecoveryDelayMillis(); int cid = builder.getClusterId(); cid = SystemPropertySupplier.create("oak.documentMK.clusterId", cid).loggingTo(LOG).get(); @@ -605,9 +608,11 @@ public final class DocumentNodeStore clusterNodeInfo = ClusterNodeInfo.getReadOnlyInstance(nonLeaseCheckingStore); } else { clusterNodeInfo = ClusterNodeInfo.getInstance(nonLeaseCheckingStore, - new RecoveryHandlerImpl(nonLeaseCheckingStore, clock, lastRevSeeker), + new RecoveryHandlerImpl(nonLeaseCheckingStore, clock, + recoveryDelayMillis, lastRevSeeker), null, null, cid, builder.isClusterInvisible(), - builder.getClusterIdReuseDelayAfterRecovery()); + builder.getClusterIdReuseDelayAfterRecovery(), + getRecoveryDelayMillis()); checkRevisionAge(nonLeaseCheckingStore, clusterNodeInfo, clock); } this.clusterId = clusterNodeInfo.getId(); @@ -2229,6 +2234,11 @@ public final class DocumentNodeStore return clock; } + @Override + public long getRecoveryDelayMillis() { + return recoveryDelayMillis; + } + @Override public String getCommitValue(@NotNull Revision changeRevision, @NotNull NodeDocument doc) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java index 69ecc068e4..907ab3a937 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java @@ -727,7 +727,7 @@ public class DocumentNodeStoreBuilder<T extends DocumentNodeStoreBuilder<T>> { } public MissingLastRevSeeker createMissingLastRevSeeker() { - return new MissingLastRevSeeker(getDocumentStore(), getClock()); + return new MissingLastRevSeeker(getDocumentStore(), getClock(), getRecoveryDelayMillis()); } public Cache<PathRev, DocumentNodeState> buildNodeCache(DocumentNodeStore store) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java index f5d24b06a5..9cd88e704e 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java @@ -524,9 +524,6 @@ public class DocumentNodeStoreService { if (isThrottlingEnabled(builder)) { builder.setThrottlingStatsCollector(new ThrottlingStatsCollectorImpl(statisticsProvider)); } - - // initialize the (global) recoveryDelayMillis - ClusterNodeInfo.setRecoveryDelayMillis(builder.getRecoveryDelayMillis()); } private boolean isWrappingCustomBlobStore() { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java index c11afa6a06..e751cf9eb3 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryAgent.java @@ -101,7 +101,7 @@ public class LastRevRecoveryAgent { public LastRevRecoveryAgent(DocumentStore store, RevisionContext context) { this(store, context, - new MissingLastRevSeeker(store, context.getClock()), + new MissingLastRevSeeker(store, context.getClock(), context.getRecoveryDelayMillis()), i -> {}); } @@ -154,7 +154,7 @@ public class LastRevRecoveryAgent { // we want to acquire. Then it's fine to go ahead with // an expired lease. ClusterNodeInfoDocument me = missingLastRevUtil.getClusterNodeInfo(revisionContext.getClusterId()); - if (me != null && me.isRecoveryNeeded(now)) { + if (me != null && me.isRecoveryNeeded(now, revisionContext.getRecoveryDelayMillis())) { String msg = String.format( "Own clusterId %s has a leaseEnd %s (%s) older than current time %s (%s). " + "Refusing to run recovery on clusterId %s.", @@ -166,7 +166,7 @@ public class LastRevRecoveryAgent { } // Check if _lastRev recovery needed for this cluster node // state is Active && current time past leaseEnd - if (nodeInfo.isRecoveryNeeded(now)) { + if (nodeInfo.isRecoveryNeeded(now, revisionContext.getRecoveryDelayMillis())) { // retrieve the root document's _lastRev NodeDocument root = missingLastRevUtil.getRoot(); Revision lastRev = root.getLastRev().get(clusterId); @@ -279,8 +279,8 @@ public class LastRevRecoveryAgent { // sweep revision. Initial sweep is not the responsibility // of the recovery agent. final RevisionContext context = new RecoveryContext(rootDoc, - revisionContext.getClock(), clusterId, - revisionContext::getCommitValue); + revisionContext.getClock(), revisionContext.getRecoveryDelayMillis(), + clusterId, revisionContext::getCommitValue); final NodeDocumentSweeper sweeper = new NodeDocumentSweeper(context, true); // make sure recovery does not run on stale cache // invalidate all suspects (OAK-9908) @@ -668,7 +668,7 @@ public class LastRevRecoveryAgent { "for id %d", clusterId); throw new DocumentStoreException(msg); } - if (!infoDoc.isRecoveryNeeded(clock.getTime())) { + if (!infoDoc.isRecoveryNeeded(clock.getTime(), revisionContext.getRecoveryDelayMillis())) { // meanwhile another process finished recovery return 0; } @@ -766,7 +766,8 @@ public class LastRevRecoveryAgent { @Override public boolean apply(ClusterNodeInfoDocument input) { return revisionContext.getClusterId() != input.getClusterId() - && input.isRecoveryNeeded(revisionContext.getClock().getTime()); + && input.isRecoveryNeeded(revisionContext.getClock().getTime(), + revisionContext.getRecoveryDelayMillis()); } }), ClusterNodeInfoDocument::getClusterId); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java index ce667f14d5..15e05e3365 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeeker.java @@ -44,9 +44,12 @@ public class MissingLastRevSeeker { protected final Clock clock; - public MissingLastRevSeeker(DocumentStore store, Clock clock) { + private final long recoveryDelayMillis; + + public MissingLastRevSeeker(DocumentStore store, Clock clock, long recoveryDelayMillis) { this.store = store; this.clock = clock; + this.recoveryDelayMillis = recoveryDelayMillis; } /** @@ -105,7 +108,7 @@ public class MissingLastRevSeeker { * @return whether the lock has been acquired */ public boolean acquireRecoveryLock(int clusterId, int recoveredBy) { - return new RecoveryLock(store, clock, clusterId) + return new RecoveryLock(store, clock, recoveryDelayMillis, clusterId) .acquireRecoveryLock(recoveredBy); } @@ -121,7 +124,7 @@ public class MissingLastRevSeeker { * @param success whether recovery was successful. */ public void releaseRecoveryLock(int clusterId, boolean success) { - new RecoveryLock(store, clock, clusterId).releaseRecoveryLock(success); + new RecoveryLock(store, clock, recoveryDelayMillis, clusterId).releaseRecoveryLock(success); } public NodeDocument getRoot() { @@ -139,7 +142,7 @@ public class MissingLastRevSeeker { public boolean isRecoveryNeeded() { long now = clock.getTime(); return StreamSupport.stream(getAllClusters().spliterator(), false) - .anyMatch(info -> info != null && info.isRecoveryNeeded(now)); + .anyMatch(info -> info != null && info.isRecoveryNeeded(now, recoveryDelayMillis)); } /** @@ -149,6 +152,6 @@ public class MissingLastRevSeeker { * instead. */ public boolean isRecoveryNeeded(@NotNull ClusterNodeInfoDocument nodeInfo) { - return nodeInfo.isRecoveryNeeded(clock.getTime()); + return nodeInfo.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis); } } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryContext.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryContext.java index c3b8ebac12..855db298e1 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryContext.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryContext.java @@ -28,6 +28,7 @@ final class RecoveryContext implements RevisionContext { private final NodeDocument root; private final Clock clock; + private final long recoveryDelayMillis; private final int clusterId; private final CommitValueResolver resolver; @@ -41,10 +42,12 @@ final class RecoveryContext implements RevisionContext { */ RecoveryContext(NodeDocument root, Clock clock, + long recoveryDelayMillis, int clusterId, CommitValueResolver resolver) { this.root = root; this.clock = clock; + this.recoveryDelayMillis = recoveryDelayMillis; this.clusterId = clusterId; this.resolver = resolver; } @@ -85,6 +88,11 @@ final class RecoveryContext implements RevisionContext { return clock; } + @Override + public long getRecoveryDelayMillis() { + return recoveryDelayMillis; + } + @Nullable @Override public String getCommitValue(@NotNull Revision changeRevision, diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerImpl.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerImpl.java index 2224a21263..680e6272d0 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerImpl.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerImpl.java @@ -41,13 +41,16 @@ class RecoveryHandlerImpl implements RecoveryHandler { private final DocumentStore store; private final Clock clock; + private final long recoveryDelayMillis; private final MissingLastRevSeeker lastRevSeeker; RecoveryHandlerImpl(DocumentStore store, Clock clock, + long recoveryDelayMillis, MissingLastRevSeeker lastRevSeeker) { this.store = store; this.clock = clock; + this.recoveryDelayMillis = recoveryDelayMillis; this.lastRevSeeker = lastRevSeeker; } @@ -66,7 +69,7 @@ class RecoveryHandlerImpl implements RecoveryHandler { NodeDocument root = Utils.getRootDocument(store); // prepare a context for recovery RevisionContext context = new RecoveryContext( - root, clock, clusterId, + root, clock, recoveryDelayMillis, clusterId, new CachingCommitValueResolver(COMMIT_VALUE_CACHE_SIZE, root::getSweepRevisions)); LastRevRecoveryAgent agent = new LastRevRecoveryAgent( store, context, lastRevSeeker, id -> {}); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLock.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLock.java index 4fc526fda9..62e5555f22 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLock.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLock.java @@ -45,6 +45,8 @@ class RecoveryLock { private final int clusterId; + private final long recoveryDelayMillis; + /** * Prepare a recovery lock on the document store for an entry with the given * {@code clusterId}. Constructing the lock does not check whether an entry @@ -54,9 +56,10 @@ class RecoveryLock { * @param clock the clock used to check whether an entry's lease expired. * @param clusterId the {@code clusterId} this lock is created for. */ - RecoveryLock(DocumentStore store, Clock clock, int clusterId) { + RecoveryLock(DocumentStore store, Clock clock, long recoveryDelayMillis, int clusterId) { this.store = store; this.clock = clock; + this.recoveryDelayMillis = recoveryDelayMillis; this.clusterId = clusterId; } @@ -75,7 +78,7 @@ class RecoveryLock { // this is unexpected... return false; } - if (!doc.isRecoveryNeeded(clock.getTime())) { + if (!doc.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)) { return false; } if (tryAcquireRecoveryLock(doc, recoveredBy)) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RevisionContext.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RevisionContext.java index 0c4c727ab7..f38d4c4570 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RevisionContext.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/RevisionContext.java @@ -59,6 +59,11 @@ public interface RevisionContext { @NotNull Clock getClock(); + /** + * @return the time in millis by which a recovery should be delayed. + */ + long getRecoveryDelayMillis(); + /** * Retrieves the commit value for a given change. This method returns the * following types of commit values: diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderBase.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderBase.java index 46f6f7decd..3e26ad9990 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderBase.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentNodeStoreBuilderBase.java @@ -192,7 +192,7 @@ public abstract class MongoDocumentNodeStoreBuilderBase<T extends MongoDocumentN public MissingLastRevSeeker createMissingLastRevSeeker() { final DocumentStore store = getDocumentStore(); if (store instanceof MongoDocumentStore) { - return new MongoMissingLastRevSeeker((MongoDocumentStore) store, getClock()); + return new MongoMissingLastRevSeeker((MongoDocumentStore) store, getClock(), getRecoveryDelayMillis()); } else { return super.createMissingLastRevSeeker(); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java index d904cc1650..aa9d251afb 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoMissingLastRevSeeker.java @@ -46,8 +46,8 @@ import org.jetbrains.annotations.NotNull; public class MongoMissingLastRevSeeker extends MissingLastRevSeeker { private final MongoDocumentStore store; - public MongoMissingLastRevSeeker(MongoDocumentStore store, Clock clock) { - super(store, clock); + public MongoMissingLastRevSeeker(MongoDocumentStore store, Clock clock, long recoveryDelayMillis) { + super(store, clock, recoveryDelayMillis); this.store = store; } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java index f997e4bf9a..1603d4c38f 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentNodeStoreBuilder.java @@ -108,7 +108,7 @@ public class RDBDocumentNodeStoreBuilder public MissingLastRevSeeker createMissingLastRevSeeker() { final DocumentStore store = getDocumentStore(); if (store instanceof RDBDocumentStore) { - return new RDBMissingLastRevSeeker((RDBDocumentStore) store, getClock()); + return new RDBMissingLastRevSeeker((RDBDocumentStore) store, getClock(), getRecoveryDelayMillis()); } else { return super.createMissingLastRevSeeker(); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBMissingLastRevSeeker.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBMissingLastRevSeeker.java index c9d9829d27..feb919b78d 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBMissingLastRevSeeker.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBMissingLastRevSeeker.java @@ -51,8 +51,8 @@ public class RDBMissingLastRevSeeker extends MissingLastRevSeeker { private final RDBDocumentStore store; - public RDBMissingLastRevSeeker(RDBDocumentStore store, Clock clock) { - super(store, clock); + public RDBMissingLastRevSeeker(RDBDocumentStore store, Clock clock, long recoveryDelayMillis) { + super(store, clock, recoveryDelayMillis); this.store = store; } diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java index 3230d97fa8..cfc79e13ae 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/ClusterNodeInfoTest.java @@ -58,6 +58,7 @@ public class ClusterNodeInfoTest { private FailureHandler handler = new FailureHandler(); private boolean invisible; private long reuseAfterRecoverMillis = ClusterNodeInfo.DEFAULT_REUSE_DELAY_AFTER_RECOVERY_MILLIS; + private long recoveryDelayMillis = ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS; public ClusterNodeInfoTest(boolean invisible) { this.invisible = invisible; @@ -79,7 +80,6 @@ public class ClusterNodeInfoTest { @After public void after() { ClusterNodeInfo.resetClockToDefault(); - ClusterNodeInfo.resetRecoveryDelayMillisToDefault(); } @Test @@ -242,7 +242,7 @@ public class ClusterNodeInfoTest { // wait until after lease end clock.waitUntil(info.getLeaseEndTime() + ClusterNodeInfo.DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS); // simulate a started recovery - MissingLastRevSeeker seeker = new MissingLastRevSeeker(store.getStore(), clock); + MissingLastRevSeeker seeker = new MissingLastRevSeeker(store.getStore(), clock, recoveryDelayMillis); assertTrue(seeker.acquireRecoveryLock(1, 42)); // cluster node 1 must not be able to renew the lease now try { @@ -735,27 +735,28 @@ public class ClusterNodeInfoTest { @Test public void recoveryNeededNoDelay() throws Exception { + recoveryDelayMillis = 0; ClusterNodeInfo info = newClusterNodeInfo(1); String key = String.valueOf(info.getId()); ClusterNodeInfoDocument doc = store.find(Collection.CLUSTER_NODES, key); - assertFalse(doc.isRecoveryNeeded(clock.getTime())); + assertFalse(doc.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); clock.waitUntil(info.getLeaseEndTime() + 1); - assertTrue(doc.isRecoveryNeeded(clock.getTime())); + assertTrue(doc.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); } @Test public void recoveryNeededWithDelay() throws Exception { - ClusterNodeInfo.setRecoveryDelayMillis(60000); + recoveryDelayMillis = 60000; ClusterNodeInfo info = newClusterNodeInfo(1); String key = String.valueOf(info.getId()); ClusterNodeInfoDocument doc = store.find(Collection.CLUSTER_NODES, key); - assertFalse(doc.isRecoveryNeeded(clock.getTime())); + assertFalse(doc.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); clock.waitUntil(info.getLeaseEndTime() + 59999); - assertFalse(doc.isRecoveryNeeded(clock.getTime())); + assertFalse(doc.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); clock.waitUntil(info.getLeaseEndTime() + 1); - assertFalse(doc.isRecoveryNeeded(clock.getTime())); + assertFalse(doc.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); clock.waitUntil(info.getLeaseEndTime() + 1); - assertTrue(doc.isRecoveryNeeded(clock.getTime())); + assertTrue(doc.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); } private void assertLeaseFailure() throws Exception { @@ -774,11 +775,11 @@ public class ClusterNodeInfoTest { clock.waitUntil(info.getLeaseEndTime() + ClusterNodeInfo.DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS); // check if expired -> recovery is needed - MissingLastRevSeeker util = new MissingLastRevSeeker(store, clock); + MissingLastRevSeeker util = new MissingLastRevSeeker(store, clock, recoveryDelayMillis); String key = String.valueOf(info.getId()); ClusterNodeInfoDocument infoDoc = store.find(Collection.CLUSTER_NODES, key); assertNotNull(infoDoc); - assertTrue(infoDoc.isRecoveryNeeded(clock.getTime())); + assertTrue(infoDoc.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); } private void recoverClusterNode(int clusterId) throws Exception { @@ -800,8 +801,8 @@ public class ClusterNodeInfoTest { private ClusterNodeInfo newClusterNodeInfo(int clusterId, String instanceId) { ClusterNodeInfo info = ClusterNodeInfo.getInstance(store, - new SimpleRecoveryHandler(store, clock), null, instanceId, clusterId, invisible, - reuseAfterRecoverMillis); + new SimpleRecoveryHandler(store, clock, recoveryDelayMillis), null, instanceId, clusterId, invisible, + reuseAfterRecoverMillis, recoveryDelayMillis); info.setLeaseFailureHandler(handler); return info; } diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceTest.java index de2f205705..87efd58991 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreServiceTest.java @@ -77,7 +77,6 @@ public class DocumentNodeStoreServiceTest { public void tearDown() throws Exception { MockOsgi.deactivate(service, context.bundleContext()); MongoUtils.dropCollections(MongoUtils.DB); - ClusterNodeInfo.resetRecoveryDelayMillisToDefault(); } @Test @@ -363,7 +362,7 @@ public class DocumentNodeStoreServiceTest { MockOsgi.activate(service, context.bundleContext()); DocumentNodeStore dns = context.getService(DocumentNodeStore.class); - assertEquals(recoveryDelayMillis, ClusterNodeInfo.getRecoveryDelayMillis()); + assertEquals(recoveryDelayMillis, dns.getRecoveryDelayMillis()); } @NotNull diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java index 87d37e8c8e..8386a84059 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java @@ -3463,7 +3463,8 @@ public class DocumentNodeStoreTest { DocumentMK.Builder nsBuilder = new DocumentMK.Builder() { @Override public MissingLastRevSeeker createMissingLastRevSeeker() { - return new MissingLastRevSeeker(getDocumentStore(), getClock()) { + return new MissingLastRevSeeker(getDocumentStore(), getClock(), + getRecoveryDelayMillis()) { @NotNull @Override public Iterable<NodeDocument> getCandidates(long startTime) { diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentSplitTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentSplitTest.java index e4d045659a..ade2e09395 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentSplitTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentSplitTest.java @@ -1144,6 +1144,11 @@ public class DocumentSplitTest extends BaseDocumentMKTest { return rc.getClock(); } + @Override + public long getRecoveryDelayMillis() { + return rc.getRecoveryDelayMillis(); + } + @Override public String getCommitValue(@NotNull Revision changeRevision, @NotNull NodeDocument doc) { diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DummyRevisionContext.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DummyRevisionContext.java index 41a465d38c..659a6dbf8a 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DummyRevisionContext.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DummyRevisionContext.java @@ -59,6 +59,11 @@ public class DummyRevisionContext implements RevisionContext { return Clock.SIMPLE; } + @Override + public long getRecoveryDelayMillis() { + return ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS; + } + @Override public String getCommitValue(@NotNull Revision changeRevision, @NotNull NodeDocument doc) { diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java index 0fbe9008cc..d591644154 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java @@ -57,6 +57,7 @@ public class LastRevRecoveryTest { public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider(); private Clock clock; + private long recoveryDelayMillis; private DocumentNodeStore ds1; private DocumentNodeStore ds2; private int c1Id; @@ -69,6 +70,7 @@ public class LastRevRecoveryTest { clock.waitUntil(System.currentTimeMillis()); Revision.setClock(clock); ClusterNodeInfo.setClock(clock); + recoveryDelayMillis = ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS; // disable lease check because we fiddle with the virtual clock final LeaseCheckMode leaseCheck = LeaseCheckMode.DISABLED; sharedStore = new MemoryDocumentStore(); @@ -194,7 +196,7 @@ public class LastRevRecoveryTest { clock.waitUntil(doc.getLeaseEndTime() + 1); // simulate ongoing recovery by cluster node 2 - MissingLastRevSeeker seeker = new MissingLastRevSeeker(sharedStore, clock); + MissingLastRevSeeker seeker = new MissingLastRevSeeker(sharedStore, clock, recoveryDelayMillis); seeker.acquireRecoveryLock(c1Id, c2Id); // run recovery from ds1 @@ -229,7 +231,7 @@ public class LastRevRecoveryTest { assertTrue(ds2.getClusterInfo().renewLease()); // simulate ongoing recovery by cluster node 2 - MissingLastRevSeeker seeker = new MissingLastRevSeeker(sharedStore, clock); + MissingLastRevSeeker seeker = new MissingLastRevSeeker(sharedStore, clock, recoveryDelayMillis); assertTrue(seeker.acquireRecoveryLock(c1Id, c2Id)); // attempt to restart ds1 while lock is acquired @@ -269,7 +271,7 @@ public class LastRevRecoveryTest { ds2.getClusterInfo().renewLease(); // start of recovery by ds2 - MissingLastRevSeeker seeker = new MissingLastRevSeeker(sharedStore, clock); + MissingLastRevSeeker seeker = new MissingLastRevSeeker(sharedStore, clock, ds1.getRecoveryDelayMillis()); assertTrue(seeker.acquireRecoveryLock(c1Id, c2Id)); // simulate crash of ds2 ClusterNodeInfoDocument info2 = sharedStore.find(CLUSTER_NODES, String.valueOf(c2Id)); @@ -283,7 +285,7 @@ public class LastRevRecoveryTest { info1 = sharedStore.find(CLUSTER_NODES, clusterId); assertNotNull(info1); - assertTrue(info1.isRecoveryNeeded(clock.getTime())); + assertTrue(info1.isRecoveryNeeded(clock.getTime(), ds1.getRecoveryDelayMillis())); assertTrue(info1.isBeingRecovered()); // restart ds1 @@ -296,7 +298,7 @@ public class LastRevRecoveryTest { .getNodeStore(); info1 = sharedStore.find(CLUSTER_NODES, clusterId); assertNotNull(info1); - assertFalse(info1.isRecoveryNeeded(clock.getTime())); + assertFalse(info1.isRecoveryNeeded(clock.getTime(), ds1.getRecoveryDelayMillis())); assertFalse(info1.isBeingRecovered()); } @@ -360,7 +362,7 @@ public class LastRevRecoveryTest { // simulate a startup with self-recovery by acquiring the clusterId // this will call the recovery handler because the lease is expired // use a seeker that takes longer than the lease duration - MissingLastRevSeeker seeker = new MissingLastRevSeeker(sharedStore, clock) { + MissingLastRevSeeker seeker = new MissingLastRevSeeker(sharedStore, clock, recoveryDelayMillis) { @Override public boolean acquireRecoveryLock(int clusterId, int recoveredBy) { assertTrue(super.acquireRecoveryLock(clusterId, recoveredBy)); @@ -375,7 +377,7 @@ public class LastRevRecoveryTest { } }; RecoveryHandler recoveryHandler = new RecoveryHandlerImpl( - sharedStore, clock, seeker); + sharedStore, clock, recoveryDelayMillis, seeker); try { // Explicitly acquiring the clusterId must fail // when it takes too long to recover diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeekerTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeekerTest.java index d33a74aa5c..c999d3e3ad 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeekerTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/MissingLastRevSeekerTest.java @@ -45,6 +45,7 @@ import static org.junit.Assert.assertTrue; public class MissingLastRevSeekerTest extends AbstractDocumentStoreTest { private Clock clock; + private long recoveryDelayMillis; private DocumentStore store; private MissingLastRevSeeker seeker; @@ -58,13 +59,14 @@ public class MissingLastRevSeekerTest extends AbstractDocumentStoreTest { clock.waitUntil(System.currentTimeMillis()); Revision.setClock(clock); ClusterNodeInfo.setClock(clock); + recoveryDelayMillis = ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS; store = ds; if (dsf == DocumentStoreFixture.MONGO) { - seeker = new MongoMissingLastRevSeeker((MongoDocumentStore) store, clock); + seeker = new MongoMissingLastRevSeeker((MongoDocumentStore) store, clock, recoveryDelayMillis); } else if (store instanceof RDBDocumentStore) { - seeker = new RDBMissingLastRevSeeker((RDBDocumentStore) store, clock); + seeker = new RDBMissingLastRevSeeker((RDBDocumentStore) store, clock, recoveryDelayMillis); } else { - seeker = new MissingLastRevSeeker(store, clock); + seeker = new MissingLastRevSeeker(store, clock, recoveryDelayMillis); } removeMeClusterNodes.add("1"); removeMeClusterNodes.add("2"); @@ -141,7 +143,7 @@ public class MissingLastRevSeekerTest extends AbstractDocumentStoreTest { assertFalse(getClusterNodeInfo(1).isBeingRecovered()); assertFalse(getClusterNodeInfo(1).isActive()); // recovery not needed anymore - assertFalse(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime())); + assertFalse(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); assertFalse(seeker.acquireRecoveryLock(1, 2)); } @@ -158,7 +160,7 @@ public class MissingLastRevSeekerTest extends AbstractDocumentStoreTest { assertFalse(getClusterNodeInfo(1).isBeingRecovered()); assertTrue(getClusterNodeInfo(1).isActive()); // recovery still needed - assertTrue(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime())); + assertTrue(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); assertTrue(seeker.acquireRecoveryLock(1, 2)); } @@ -171,15 +173,15 @@ public class MissingLastRevSeekerTest extends AbstractDocumentStoreTest { ClusterNodeInfo.getInstance(store, NOOP, null, null, 2); assertTrue(seeker.isRecoveryNeeded()); - assertTrue(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime())); - assertFalse(getClusterNodeInfo(2).isRecoveryNeeded(clock.getTime())); + assertTrue(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); + assertFalse(getClusterNodeInfo(2).isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); assertTrue(seeker.acquireRecoveryLock(1, 2)); seeker.releaseRecoveryLock(1, true); assertFalse(seeker.isRecoveryNeeded()); - assertFalse(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime())); - assertFalse(getClusterNodeInfo(2).isRecoveryNeeded(clock.getTime())); + assertFalse(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); + assertFalse(getClusterNodeInfo(2).isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); } @Test @@ -193,12 +195,12 @@ public class MissingLastRevSeekerTest extends AbstractDocumentStoreTest { assertTrue(seeker.acquireRecoveryLock(1, 2)); assertTrue(seeker.isRecoveryNeeded()); - assertTrue(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime())); + assertTrue(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); seeker.releaseRecoveryLock(1, true); assertFalse(seeker.isRecoveryNeeded()); - assertFalse(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime())); + assertFalse(getClusterNodeInfo(1).isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); } diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeperIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeperIT.java index 42e397992e..1b31ce65ab 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeperIT.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentSweeperIT.java @@ -48,6 +48,8 @@ public class NodeDocumentSweeperIT extends AbstractTwoNodeTest { private LastRevRecoveryAgent agent2; + private long recoveryDelayMillis; + public NodeDocumentSweeperIT(DocumentStoreFixture fixture) { super(fixture); } @@ -64,12 +66,13 @@ public class NodeDocumentSweeperIT extends AbstractTwoNodeTest { @Before public void prepareAgent() { + recoveryDelayMillis = ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS; // first setup seeker according to underlying document store implementation MissingLastRevSeeker seeker; if (store2 instanceof MongoDocumentStore) { - seeker = new MongoMissingLastRevSeeker((MongoDocumentStore) store2, clock); + seeker = new MongoMissingLastRevSeeker((MongoDocumentStore) store2, clock, recoveryDelayMillis); } else if (store2 instanceof RDBDocumentStore) { - seeker = new RDBMissingLastRevSeeker((RDBDocumentStore) store2, clock) { + seeker = new RDBMissingLastRevSeeker((RDBDocumentStore) store2, clock, recoveryDelayMillis) { @Override public @NotNull Iterable<NodeDocument> getCandidates(long startTime) { List<NodeDocument> docs = new ArrayList<>(); @@ -80,11 +83,11 @@ public class NodeDocumentSweeperIT extends AbstractTwoNodeTest { }; } else { // use default implementation - seeker = new MissingLastRevSeeker(store2, clock); + seeker = new MissingLastRevSeeker(store2, clock, recoveryDelayMillis); } // then customize seeker to return documents in a defined order // return docs sorted by decreasing depth - MissingLastRevSeeker testSeeker = new MissingLastRevSeeker(store2, clock) { + MissingLastRevSeeker testSeeker = new MissingLastRevSeeker(store2, clock, recoveryDelayMillis) { @Override public @NotNull Iterable<NodeDocument> getCandidates(long startTime) { List<NodeDocument> docs = new ArrayList<>(); diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryContextTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryContextTest.java index 097ce9c938..2046c96af9 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryContextTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryContextTest.java @@ -33,10 +33,24 @@ public class RecoveryContextTest { int clusterId = 1; RevisionContext context = new RecoveryContext(doc, Clock.SIMPLE, + ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS, clusterId, (r, d) -> null); assertEquals(clusterId, context.getClusterId()); + assertEquals(ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS, context.getRecoveryDelayMillis()); assertEquals(0, context.getBranches().size()); assertThat(context.getPendingModifications().getPaths(), empty()); assertEquals(clusterId, context.newRevision().getClusterId()); } + + @Test + public void recoveryDelay() { + DocumentStore store = new MemoryDocumentStore(); + NodeDocument doc = new NodeDocument(store); + int clusterId = 1; + + RevisionContext context = new RecoveryContext(doc, Clock.SIMPLE, + 42, + clusterId, (r, d) -> null); + assertEquals(42, context.getRecoveryDelayMillis()); + } } diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerTest.java index 97acc2dae2..7ba2e9340e 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryHandlerTest.java @@ -35,9 +35,10 @@ public class RecoveryHandlerTest { public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider(); private Clock clock = new Clock.Virtual(); + private long recoveryDelayMillis = ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS; private FailingDocumentStore store = new FailingDocumentStore(new MemoryDocumentStore()); - private MissingLastRevSeeker seeker = new MissingLastRevSeeker(store, clock); - private RecoveryHandler handler = new RecoveryHandlerImpl(store, clock, seeker); + private MissingLastRevSeeker seeker = new MissingLastRevSeeker(store, clock, recoveryDelayMillis); + private RecoveryHandler handler = new RecoveryHandlerImpl(store, clock, recoveryDelayMillis, seeker); @Before public void before() throws Exception { diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLockTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLockTest.java index 255e887bdc..76d35c7475 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLockTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/RecoveryLockTest.java @@ -51,10 +51,12 @@ public class RecoveryLockTest { private Clock clock = new Clock.Virtual(); + private long recoveryDelayMillis = ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS; + private ExecutorService executor = Executors.newCachedThreadPool(); - private RecoveryLock lock1 = new RecoveryLock(store, clock, 1); - private RecoveryLock lock2 = new RecoveryLock(store, clock, 2); + private RecoveryLock lock1 = new RecoveryLock(store, clock, recoveryDelayMillis, 1); + private RecoveryLock lock2 = new RecoveryLock(store, clock, recoveryDelayMillis, 2); private ClusterNodeInfo info1; @@ -143,8 +145,8 @@ public class RecoveryLockTest { // expire clusterId 1 clock.waitUntil(info1.getLeaseEndTime() + DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS); ClusterNodeInfoDocument c = infoDocument(1); - MissingLastRevSeeker seeker = new MissingLastRevSeeker(store, clock); - assertTrue(c.isRecoveryNeeded(clock.getTime())); + MissingLastRevSeeker seeker = new MissingLastRevSeeker(store, clock, recoveryDelayMillis); + assertTrue(c.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); assertFalse(c.isBeingRecovered()); Semaphore recovering = new Semaphore(0); @@ -163,7 +165,7 @@ public class RecoveryLockTest { // check state again c = infoDocument(1); - assertTrue(c.isRecoveryNeeded(clock.getTime())); + assertTrue(c.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); assertTrue(c.isBeingRecovered()); assertTrue(c.isBeingRecoveredBy(1)); // clusterId 2 must not be able to acquire (break) the recovery lock @@ -176,7 +178,7 @@ public class RecoveryLockTest { // check state again c = infoDocument(1); - assertFalse(c.isRecoveryNeeded(clock.getTime())); + assertFalse(c.isRecoveryNeeded(clock.getTime(), recoveryDelayMillis)); assertFalse(c.isBeingRecovered()); assertFalse(c.isBeingRecoveredBy(1)); @@ -194,7 +196,7 @@ public class RecoveryLockTest { info1 = ClusterNodeInfo.getInstance(store, RecoveryHandler.NOOP, null, "node1", 1); - RecoveryLock recLock = new RecoveryLock(store, clock, 1); + RecoveryLock recLock = new RecoveryLock(store, clock, recoveryDelayMillis, 1); // expire clusterId 1 clock.waitUntil(info1.getLeaseEndTime() + DEFAULT_LEASE_UPDATE_INTERVAL_MILLIS); diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SimpleRecoveryHandler.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SimpleRecoveryHandler.java index 2a6c567541..0d47ce4e0b 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SimpleRecoveryHandler.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/SimpleRecoveryHandler.java @@ -31,16 +31,20 @@ public class SimpleRecoveryHandler implements RecoveryHandler { private final Clock clock; + private final long recoveryDelayMillis; + public SimpleRecoveryHandler(@NotNull DocumentStore store, - @NotNull Clock clock) { + @NotNull Clock clock, + long recoveryDelayMillis) { this.store = checkNotNull(store); this.clock = checkNotNull(clock); + this.recoveryDelayMillis = recoveryDelayMillis; } @Override public boolean recover(int clusterId) { // simulate recovery by acquiring recovery lock - RecoveryLock lock = new RecoveryLock(store, clock, clusterId); + RecoveryLock lock = new RecoveryLock(store, clock, recoveryDelayMillis, clusterId); if (lock.acquireRecoveryLock(clusterId)) { lock.releaseRecoveryLock(true); return true; diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/AcquireRecoveryLockTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/AcquireRecoveryLockTest.java index db79a5dc0d..c0ee935aa0 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/AcquireRecoveryLockTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/AcquireRecoveryLockTest.java @@ -41,6 +41,8 @@ public class AcquireRecoveryLockTest extends AbstractMongoConnectionTest { private Clock clock = new Clock.Virtual(); + private long recoveryDelayMillis = ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS; + private MongoDocumentStore store; @Before @@ -72,7 +74,7 @@ public class AcquireRecoveryLockTest extends AbstractMongoConnectionTest { // OAK-4131 @Test public void recoveryBy() throws Exception { - MongoMissingLastRevSeeker seeker = new MongoMissingLastRevSeeker(store, getTestClock()); + MongoMissingLastRevSeeker seeker = new MongoMissingLastRevSeeker(store, getTestClock(), recoveryDelayMillis); List<ClusterNodeInfoDocument> infoDocs = newArrayList(seeker.getAllClusters()); assertEquals(1, infoDocs.size()); int clusterId = infoDocs.get(0).getClusterId(); diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java index bb04c6ab8b..87db76a636 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/LeaseUpdateSocketTimeoutIT.java @@ -78,6 +78,8 @@ public class LeaseUpdateSocketTimeoutIT { private Clock clock; + private long recoveryDelayMillis; + private DocumentStore store; private final FailureHandler handler = new FailureHandler(); @@ -93,6 +95,7 @@ public class LeaseUpdateSocketTimeoutIT { clock = new Clock.Virtual(); clock.waitUntil(System.currentTimeMillis()); setClusterNodeInfoClock(clock); + recoveryDelayMillis = ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS; ToxiproxyClient toxiproxyClient = new ToxiproxyClient(tp.getHost(), tp.getControlPort()); proxy = toxiproxyClient.createProxy("mongo", "0.0.0.0:8666", "mongo:" + MONGODB_DEFAULT_PORT); String uri = "mongodb://" + tp.getHost() + ":" + tp.getMappedPort(8666); @@ -157,7 +160,7 @@ public class LeaseUpdateSocketTimeoutIT { private ClusterNodeInfo newClusterNodeInfo() { ClusterNodeInfo info = ClusterNodeInfo.getInstance(store, - new SimpleRecoveryHandler(store, clock), null, null, 1); + new SimpleRecoveryHandler(store, clock, recoveryDelayMillis), null, null, 1); info.setLeaseFailureHandler(handler); return info; } diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java index 7e5500903c..6f65beb755 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreTest.java @@ -34,6 +34,7 @@ import java.util.Set; import org.apache.jackrabbit.oak.commons.junit.LogCustomizer; import org.apache.jackrabbit.oak.plugins.document.AbstractDocumentStoreTest; +import org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo; import org.apache.jackrabbit.oak.plugins.document.Collection; import org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture; import org.apache.jackrabbit.oak.plugins.document.MissingLastRevSeeker; @@ -52,6 +53,8 @@ import org.apache.jackrabbit.guava.common.collect.Sets; public class RDBDocumentStoreTest extends AbstractDocumentStoreTest { + private long recoveryDelayMillis = ClusterNodeInfo.DEFAULT_RECOVERY_DELAY_MILLIS; + public RDBDocumentStoreTest(DocumentStoreFixture dsf) { super(dsf); } @@ -212,7 +215,7 @@ public class RDBDocumentStoreTest extends AbstractDocumentStoreTest { Set<String> ids = Sets.newHashSet(); boolean updated = false; - MissingLastRevSeeker seeker = new RDBMissingLastRevSeeker((RDBDocumentStore) ds, Clock.SIMPLE); + MissingLastRevSeeker seeker = new RDBMissingLastRevSeeker((RDBDocumentStore) ds, Clock.SIMPLE, recoveryDelayMillis); for (NodeDocument doc : seeker.getCandidates(0)) { if (!updated) { // as soon as we have the first document, update
