Author: chetanm
Date: Thu Mar 27 15:18:49 2014
New Revision: 1582346
URL: http://svn.apache.org/r1582346
Log:
OAK-1341 - DocumentNodeStore: Implement revision garbage collection (WIP)
Add method for batch removal to DocumentStore
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoDocumentStoreTest.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java?rev=1582346&r1=1582345&r2=1582346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentStore.java
Thu Mar 27 15:18:49 2014
@@ -114,6 +114,15 @@ public interface DocumentStore {
<T extends Document> void remove(Collection<T> collection, String key);
/**
+ * Batch remove documents with given key.
+ *
+ * @param <T> the document type
+ * @param collection the collection
+ * @param keys list of keys
+ */
+ <T extends Document> void remove(Collection<T> collection, List<String>
keys);
+
+ /**
* Try to create a list of documents.
*
* @param <T> the document type
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java?rev=1582346&r1=1582345&r2=1582346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java
Thu Mar 27 15:18:49 2014
@@ -150,6 +150,13 @@ public class MemoryDocumentStore impleme
}
}
+ @Override
+ public <T extends Document> void remove(Collection<T> collection,
List<String> keys) {
+ for(String key : keys){
+ remove(collection, key);
+ }
+ }
+
@CheckForNull
@Override
public <T extends Document> T createOrUpdate(Collection<T> collection,
UpdateOp update)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java?rev=1582346&r1=1582345&r2=1582346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoDocumentStore.java
Thu Mar 27 15:18:49 2014
@@ -36,6 +36,7 @@ import javax.annotation.Nullable;
import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
import org.apache.jackrabbit.mk.api.MicroKernelException;
import org.apache.jackrabbit.oak.cache.CacheStats;
import org.apache.jackrabbit.oak.cache.CacheValue;
@@ -86,6 +87,8 @@ public class MongoDocumentStore implemen
private static final DBObject BY_ID_ASC = new BasicDBObject(Document.ID,
1);
+ public static final int IN_CLAUSE_BATCH_SIZE = 500;
+
private final DBCollection nodes;
private final DBCollection clusterNodes;
private final DBCollection settings;
@@ -204,6 +207,13 @@ public class MongoDocumentStore implemen
}
}
+ public <T extends Document> void invalidateCache(Collection<T> collection,
List<String> keys) {
+ for(String key : keys){
+ invalidateCache(collection, key);
+ }
+ }
+
+
@Override
public <T extends Document> T find(Collection<T> collection, String key) {
return find(collection, key, Integer.MAX_VALUE);
@@ -380,6 +390,20 @@ public class MongoDocumentStore implemen
}
}
+ @Override
+ public <T extends Document> void remove(Collection<T> collection,
List<String> keys) {
+ DBCollection dbCollection = getDBCollection(collection);
+ for(List<String> keyBatch : Lists.partition(keys,
IN_CLAUSE_BATCH_SIZE)){
+ DBObject query =
QueryBuilder.start(Document.ID).in(keyBatch).get();
+ WriteResult writeResult = dbCollection.remove(query,
WriteConcern.SAFE);
+ invalidateCache(collection, keyBatch);
+ if (writeResult.getError() != null) {
+ throw new MicroKernelException("Remove failed: " +
writeResult.getError());
+ }
+ }
+
+ }
+
@CheckForNull
private <T extends Document> T findAndModify(Collection<T> collection,
UpdateOp updateOp,
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java?rev=1582346&r1=1582345&r2=1582346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
Thu Mar 27 15:18:49 2014
@@ -167,6 +167,14 @@ public class RDBDocumentStore implements
}
@Override
+ public <T extends Document> void remove(Collection<T> collection,
List<String> keys) {
+ //TODO Use batch delete
+ for(String key : keys){
+ remove(collection, key);
+ }
+ }
+
+ @Override
public <T extends Document> boolean create(Collection<T> collection,
List<UpdateOp> updateOps) {
return internalCreate(collection, updateOps);
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java?rev=1582346&r1=1582345&r2=1582346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/LoggingDocumentStoreWrapper.java
Thu Mar 27 15:18:49 2014
@@ -140,6 +140,14 @@ public class LoggingDocumentStoreWrapper
}
@Override
+ public <T extends Document> void remove(Collection<T> collection,
List<String> keys) {
+ //TODO Logging
+ for(String key : keys){
+ remove(collection, key);
+ }
+ }
+
+ @Override
public <T extends Document> boolean create(final Collection<T> collection,
final List<UpdateOp> updateOps)
{
try {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java?rev=1582346&r1=1582345&r2=1582346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/SynchronizingDocumentStoreWrapper.java
Thu Mar 27 15:18:49 2014
@@ -68,6 +68,13 @@ public class SynchronizingDocumentStoreW
}
@Override
+ public synchronized <T extends Document> void remove(Collection<T>
collection, List<String> keys) {
+ for(String key : keys){
+ remove(collection, key);
+ }
+ }
+
+ @Override
public synchronized <T extends Document> boolean create(final
Collection<T> collection, final List<UpdateOp> updateOps) {
return store.create(collection, updateOps);
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java?rev=1582346&r1=1582345&r2=1582346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/TimingDocumentStoreWrapper.java
Thu Mar 27 15:18:49 2014
@@ -177,6 +177,14 @@ public class TimingDocumentStoreWrapper
}
@Override
+ public <T extends Document> void remove(Collection<T> collection,
List<String> keys) {
+ //TODO Timing
+ for(String key : keys){
+ remove(collection, key);
+ }
+ }
+
+ @Override
public <T extends Document> boolean create(Collection<T> collection,
List<UpdateOp> updateOps) {
try {
long start = now();
Modified:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoDocumentStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoDocumentStoreTest.java?rev=1582346&r1=1582345&r2=1582346&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoDocumentStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoDocumentStoreTest.java
Thu Mar 27 15:18:49 2014
@@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorServ
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
@@ -105,6 +106,35 @@ public class MongoDocumentStoreTest {
}
@Test
+ public void batchRemove() throws Exception {
+ DocumentStore docStore = openDocumentStore();
+ int nUpdates = 10;
+ Revision r1 = new Revision(0, 0, 0);
+ List<String> ids = Lists.newArrayList();
+ List<UpdateOp> updateOps = new ArrayList<UpdateOp>();
+ for (int i = 0; i < nUpdates; i++) {
+ String path = "/node" + i;
+ UpdateOp updateOp = new UpdateOp(path, true);
+ updateOp.set(Document.ID, "/node" + i);
+ updateOp.setMapEntry("property1", r1, "value1");
+ updateOp.increment("property2", 1);
+ updateOp.set("property3", "value3");
+ updateOps.add(updateOp);
+ ids.add(updateOp.getId());
+ }
+ docStore.create(Collection.NODES, updateOps);
+
+ for(String id : ids){
+ assertNotNull(docStore.find(Collection.NODES, id));
+ }
+
+ docStore.remove(Collection.NODES, ids);
+ for(String id : ids){
+ assertNull(docStore.find(Collection.NODES, id));
+ }
+ }
+
+ @Test
public void batchAdd() throws Exception {
DocumentStore docStore = openDocumentStore();
Revision r1 = new Revision(0, 0, 0);
