Author: chetanm
Date: Mon Mar 31 15:06:12 2014
New Revision: 1583344
URL: http://svn.apache.org/r1583344
Log:
OAK-1295 - Recovery for missing _lastRev updates (WIP)
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecovery.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecovery.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecovery.java?rev=1583344&r1=1583343&r2=1583344&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecovery.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecovery.java
Mon Mar 31 15:06:12 2014
@@ -20,17 +20,12 @@
package org.apache.jackrabbit.oak.plugins.document;
import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
import java.util.concurrent.locks.ReentrantLock;
import javax.annotation.CheckForNull;
import com.google.common.base.Predicate;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import org.apache.jackrabbit.oak.commons.PathUtils;
-import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,77 +42,44 @@ public class LastRevRecovery {
this.nodeStore = nodeStore;
}
- public void recover(Iterator<NodeDocument> suspects, int clusterId) {
+ public int recover(Iterator<NodeDocument> suspects, int clusterId) {
UnsavedModifications unsaved = new UnsavedModifications();
- //Set of parent path whose lastRev has been updated based on
- //last rev information obtained from suspects. Its possible
- //that lastRev for such parents present in DS has
- //higher value. So before persisting the changes for these
- //paths we need to ensure that there actual lastRev is lesser
- //than one being set via unsaved
- Set<String> unverifiedParentPaths = Sets.newHashSet();
-
- //Map of known last rev of checked paths
- Map<String, Revision> knownLastRevs = Maps.newHashMap();
-
while (suspects.hasNext()) {
NodeDocument doc = suspects.next();
Revision currentLastRev = doc.getLastRev().get(clusterId);
- if (currentLastRev != null) {
- knownLastRevs.put(doc.getPath(), currentLastRev);
- }
-
Revision lostLastRev = determineMissedLastRev(doc, clusterId);
- //lastRev is consistent
- if (lostLastRev == null) {
- continue;
- }
-
//1. Update lastRev for this doc
- unsaved.put(doc.getPath(), lostLastRev);
-
- //2. Update lastRev for parent paths
- String path = doc.getPath();
- while (true) {
- if (PathUtils.denotesRoot(path)) {
- break;
- }
- path = PathUtils.getParentPath(path);
- unsaved.put(path, lostLastRev);
- unverifiedParentPaths.add(path);
+ if (lostLastRev != null) {
+ unsaved.put(doc.getPath(), lostLastRev);
}
- }
- //By now we have iterated over all suspects so remove entries for paths
- //whose lastRev have been determined on the basis of state obtained
from
- //DS
- Iterator<String> unverifiedParentPathsItr =
unverifiedParentPaths.iterator();
- while (unverifiedParentPathsItr.hasNext()) {
- String unverifiedParentPath = unverifiedParentPathsItr.next();
- Revision knownRevision = knownLastRevs.get(unverifiedParentPath);
- if (knownRevision != null) {
- unverifiedParentPathsItr.remove();
- unsaved.put(unverifiedParentPath, knownRevision);
- }
- }
+ Revision lastRevForParents = lostLastRev != null ? lostLastRev :
currentLastRev;
- //Now for the left over unverifiedParentPaths determine the lastRev
- //from DS and add them to unsaved. This ensures that we do not set
lastRev
- //to a lower value
-
- //TODO For Mongo case we can fetch such documents more efficiently
- //via batch fetch
- for (String path : unverifiedParentPaths) {
- NodeDocument doc = getDocument(path);
- if (doc != null) {
- Revision lastRev = doc.getLastRev().get(clusterId);
- unsaved.put(path, lastRev);
+ //If both currentLastRev and lostLastRev are null it means
+ //that no change is done by suspect cluster on this document
+ //so nothing needs to be updated. Probably it was only changed by
+ //other cluster nodes. If this node is parent of any child node
which
+ //has been modified by cluster then that node roll up would
+ //add this node path to unsaved
+
+ //2. Update lastRev for parent paths aka rollup
+ if (lastRevForParents != null) {
+ String path = doc.getPath();
+ while (true) {
+ if (PathUtils.denotesRoot(path)) {
+ break;
+ }
+ path = PathUtils.getParentPath(path);
+ unsaved.put(path, lastRevForParents);
+ }
}
}
+ //Note the size before persist as persist operation
+ //would empty the internal state
int size = unsaved.getPaths().size();
if (log.isDebugEnabled()) {
@@ -131,6 +93,8 @@ public class LastRevRecovery {
log.info("Updated lastRev of [{}] documents while performing lastRev
recovery for " +
"cluster node [{}]", size, clusterId);
+
+ return size;
}
/**
@@ -154,9 +118,8 @@ public class LastRevRecovery {
//Merge sort the revs for which changes have been made
//to this doc
- //TODO Would looking into the Local map be sufficient
- //Probably yes as entries for a particular cluster node
- //are split by that cluster only
+ //localMap always keeps the most recent valid commit entry
+ //per cluster node so looking into that should be sufficient
Iterable<Revision> revs = mergeSorted(of(
filter(doc.getLocalCommitRoot().keySet(), cp),
filter(doc.getLocalRevisions().keySet(), cp)),
@@ -180,10 +143,6 @@ public class LastRevRecovery {
return null;
}
- private NodeDocument getDocument(String path) {
- return nodeStore.getDocumentStore().find(Collection.NODES,
Utils.getIdFromPath(path));
- }
-
private static class ClusterPredicate implements Predicate<Revision> {
private final int clusterId;
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java?rev=1583344&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java
Mon Mar 31 15:06:12 2014
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.document;
+
+import com.google.common.collect.Iterators;
+import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+public class LastRevRecoveryTest {
+ private DocumentNodeStore ds1;
+ private DocumentNodeStore ds2;
+ private MemoryDocumentStore sharedStore;
+
+ @Before
+ public void setUp(){
+ sharedStore = new MemoryDocumentStore();
+ ds1 = new DocumentMK.Builder()
+ .setClusterId(1)
+ .setAsyncDelay(0)
+ .setDocumentStore(sharedStore)
+ .getNodeStore();
+
+ ds2 = new DocumentMK.Builder()
+ .setClusterId(2)
+ .setAsyncDelay(0)
+ .setDocumentStore(sharedStore)
+ .getNodeStore();
+ }
+
+
+ @Test
+ public void testRecover() throws Exception {
+ //1. Create base structure /x/y
+ NodeBuilder b1 = ds1.getRoot().builder();
+ b1.child("x").child("y");
+ ds1.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ ds1.runBackgroundOperations();
+
+ //lastRev are persisted directly for new nodes. In case of
+ // updates they are persisted via background jobs
+
+ //1.2 Get last rev populated for root node for ds2
+ ds2.runBackgroundOperations();
+ NodeBuilder b2 = ds2.getRoot().builder();
+ b2.child("x").setProperty("f1","b1");
+ ds2.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ ds2.runBackgroundOperations();
+
+ //2. Add a new node /x/y/z
+ b2 = ds2.getRoot().builder();
+ b2.child("x").child("y").child("z").setProperty("foo", "bar");
+ ds2.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ //Refresh DS1
+ ds1.runBackgroundOperations();
+
+ NodeDocument z1 = getDocument(ds1, "/x/y/z");
+ NodeDocument y1 = getDocument(ds1, "/x/y");
+ NodeDocument x1 = getDocument(ds1, "/x");
+
+ Revision zlastRev2 = z1.getLastRev().get(2);
+ assertNotNull(zlastRev2);
+
+ //lastRev should not be updated for C #2
+ assertNull(y1.getLastRev().get(2));
+
+ LastRevRecovery recovery = new LastRevRecovery(ds1);
+
+ //Do not pass y1 but still y1 should be updated
+ recovery.recover(Iterators.forArray(x1,z1), 2);
+
+ //Post recovery the lastRev should be updated for /x/y and /x
+ assertEquals(zlastRev2, getDocument(ds1, "/x/y").getLastRev().get(2));
+ assertEquals(zlastRev2, getDocument(ds1, "/x").getLastRev().get(2));
+ assertEquals(zlastRev2, getDocument(ds1, "/").getLastRev().get(2));
+ }
+
+ private NodeDocument getDocument(DocumentNodeStore nodeStore, String path)
{
+ return nodeStore.getDocumentStore().find(Collection.NODES,
Utils.getIdFromPath(path));
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/LastRevRecoveryTest.java
------------------------------------------------------------------------------
svn:eol-style = native
