Author: thomasm
Date: Wed Mar 19 08:55:44 2014
New Revision: 1579175
URL: http://svn.apache.org/r1579175
Log:
OAK-1510 MongoDB / DocumentNodeStore DataStore GC performance (patch 2)
Modified:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
Modified:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java?rev=1579175&r1=1579174&r2=1579175&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/FileBlobStore.java
Wed Mar 19 08:55:44 2014
@@ -27,6 +27,7 @@ import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Iterator;
+import java.util.List;
import javax.annotation.Nullable;
@@ -219,20 +220,22 @@ public class FileBlobStore extends Abstr
}
@Override
- public boolean deleteChunk(String chunkId, long maxLastModifiedTime)
throws Exception {
- byte[] digest = StringUtils.convertHexToBytes(chunkId);
- File f = getFile(digest, false);
- if (!f.exists()) {
- File old = getFile(digest, true);
- f.getParentFile().mkdir();
- old.renameTo(f);
- f = getFile(digest, false);
- }
- if ((maxLastModifiedTime <= 0)
- || FileUtils.isFileOlder(f, maxLastModifiedTime)) {
- return f.delete();
+ public boolean deleteChunks(List<String> chunkIds, long
maxLastModifiedTime) throws Exception {
+ for (String chunkId : chunkIds) {
+ byte[] digest = StringUtils.convertHexToBytes(chunkId);
+ File f = getFile(digest, false);
+ if (!f.exists()) {
+ File old = getFile(digest, true);
+ f.getParentFile().mkdir();
+ old.renameTo(f);
+ f = getFile(digest, false);
+ }
+ if ((maxLastModifiedTime <= 0)
+ || FileUtils.isFileOlder(f, maxLastModifiedTime)) {
+ f.delete();
+ }
}
- return false;
+ return true;
}
@Override
Modified:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java?rev=1579175&r1=1579174&r2=1579175&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
Wed Mar 19 08:55:44 2014
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.spi.bl
import java.io.IOException;
import java.util.Iterator;
+import java.util.List;
/**
* A blob store that support garbage collection.
@@ -83,16 +84,14 @@ public interface GarbageCollectableBlobS
Iterator<String> getAllChunkIds(long maxLastModifiedTime) throws Exception;
/**
- * Delete the blob with the given id.
- *
- * @param chunkId the chunk id
- * @param maxLastModifiedTime
- * the max last modified time to consider for retrieval
+ * Deletes the blobs with the given ids.
+ *
+ * @param chunkIds the chunk ids
+ * @param maxLastModifiedTime the max last modified time to consider for
retrieval
* @return true, if successful
- * @throws Exception
- * the exception
+ * @throws Exception the exception
*/
- boolean deleteChunk(String chunkId, long maxLastModifiedTime) throws
Exception;
+ boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime)
throws Exception;
/**
* Resolve chunks from the given Id.
Modified:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java?rev=1579175&r1=1579174&r2=1579175&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/MemoryBlobStore.java
Wed Mar 19 08:55:44 2014
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.spi.bl
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Maps;
@@ -84,12 +85,14 @@ public class MemoryBlobStore extends Abs
* Ignores the maxlastModifiedTime
*/
@Override
- public boolean deleteChunk(String chunkId, long maxLastModifiedTime)
throws Exception {
- BlockId id = new BlockId(StringUtils.convertHexToBytes(chunkId), 0);
- if (map.containsKey(id)) {
- map.remove(id);
- } else if (old.containsKey(id)) {
- old.remove(id);
+ public boolean deleteChunks(List<String> chunkIds, long
maxLastModifiedTime) throws Exception {
+ for (String chunkId : chunkIds) {
+ BlockId id = new BlockId(StringUtils.convertHexToBytes(chunkId),
0);
+ if (map.containsKey(id)) {
+ map.remove(id);
+ } else if (old.containsKey(id)) {
+ old.remove(id);
+ }
}
return true;
}
Modified:
jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java?rev=1579175&r1=1579174&r2=1579175&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob/src/test/java/org/apache/jackrabbit/oak/spi/blob/AbstractBlobStoreTest.java
Wed Mar 19 08:55:44 2014
@@ -31,6 +31,7 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
@@ -394,10 +395,8 @@ public abstract class AbstractBlobStoreT
@Test
public void delete() throws Exception {
Set<String> ids = createArtifacts();
-
- for (String id : ids) {
- store.deleteChunk(id, 0);
- }
+
+ store.deleteChunks(Lists.newArrayList(ids), 0);
Iterator<String> iter = store.getAllChunkIds(0);
Set<String> ret = Sets.newHashSet();
@@ -405,7 +404,7 @@ public abstract class AbstractBlobStoreT
ret.add(iter.next());
}
- assertTrue(ret.isEmpty());
+ assertTrue(ret.toString(), ret.isEmpty());
}
private Set<String> createArtifacts() throws Exception {
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1579175&r1=1579174&r2=1579175&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Wed Mar 19 08:55:44 2014
@@ -416,17 +416,15 @@ public class MarkSweepGarbageCollector i
@Override
public void run() {
- for (String id : ids) {
- try {
- boolean deleted = ((GarbageCollectableBlobStore)
nodeStore.getBlobStore())
- .deleteChunk(id, maxLastModifiedTime);
- if (!deleted) {
- exceptionQueue.add(id);
- }
- } catch (Exception e) {
- e.printStackTrace();
- exceptionQueue.add(id);
+ try {
+ boolean deleted = ((GarbageCollectableBlobStore)
nodeStore.getBlobStore())
+ .deleteChunks(ids, maxLastModifiedTime);
+ if (!deleted) {
+ exceptionQueue.addAll(ids);
}
+ } catch (Exception e) {
+ e.printStackTrace();
+ exceptionQueue.addAll(ids);
}
}
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java?rev=1579175&r1=1579174&r2=1579175&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/cloud/CloudBlobStore.java
Wed Mar 19 08:55:44 2014
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -226,18 +227,20 @@ public class CloudBlobStore extends Cach
final org.jclouds.blobstore.BlobStore blobStore =
context.getBlobStore();
return new CloudStoreIterator(blobStore, maxLastModifiedTime);
-}
+ }
@Override
- public boolean deleteChunk(String chunkId, long maxLastModifiedTime)
throws Exception {
+ public boolean deleteChunks(List<String> chunkIds, long
maxLastModifiedTime) throws Exception {
Preconditions.checkNotNull(context);
- final org.jclouds.blobstore.BlobStore blobStore =
context.getBlobStore();
- StorageMetadata metadata = blobStore.blobMetadata(cloudContainer,
chunkId);
- if ((maxLastModifiedTime <= 0)
- || (metadata.getLastModified().getTime() <=
maxLastModifiedTime)) {
- blobStore.removeBlob(cloudContainer, chunkId);
- return true;
+ for (String chunkId : chunkIds) {
+ final org.jclouds.blobstore.BlobStore blobStore =
context.getBlobStore();
+ StorageMetadata metadata = blobStore.blobMetadata(cloudContainer,
chunkId);
+ if ((maxLastModifiedTime <= 0)
+ || (metadata.getLastModified().getTime() <=
maxLastModifiedTime)) {
+ blobStore.removeBlob(cloudContainer, chunkId);
+ return true;
+ }
}
return true;
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1579175&r1=1579174&r2=1579175&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
Wed Mar 19 08:55:44 2014
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.SequenceInputStream;
import java.util.Iterator;
+import java.util.List;
import javax.annotation.Nullable;
import javax.jcr.RepositoryException;
@@ -228,17 +229,20 @@ public class DataStoreBlobStore implemen
}
@Override
- public boolean deleteChunk(String chunkId, long maxLastModifiedTime)
throws Exception {
+ public boolean deleteChunks(List<String> chunkIds, long
maxLastModifiedTime) throws Exception {
if (delegate instanceof MultiDataStoreAware) {
- DataIdentifier identifier = new DataIdentifier(chunkId);
- DataRecord dataRecord = delegate.getRecord(identifier);
- if ((maxLastModifiedTime <= 0)
- || dataRecord.getLastModified() <= maxLastModifiedTime) {
+ for (String chunkId : chunkIds) {
+ DataIdentifier identifier = new DataIdentifier(chunkId);
+ DataRecord dataRecord = delegate.getRecord(identifier);
+ boolean success = (maxLastModifiedTime <= 0)
+ || dataRecord.getLastModified() <= maxLastModifiedTime;
+ if (!success) {
+ return false;
+ }
((MultiDataStoreAware) delegate).deleteRecord(identifier);
- return true;
}
}
- return false;
+ return true;
}
@Override
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java?rev=1579175&r1=1579174&r2=1579175&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/db/DbBlobStore.java
Wed Mar 19 08:55:44 2014
@@ -24,6 +24,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import org.apache.jackrabbit.oak.commons.StringUtils;
import org.apache.jackrabbit.oak.plugins.blob.CachingBlobStore;
@@ -220,32 +221,49 @@ public class DbBlobStore extends Caching
}
@Override
- public boolean deleteChunk(String chunkId, long maxLastModifiedTime)
throws Exception {
+ public boolean deleteChunks(List<String> chunkIds, long
maxLastModifiedTime) throws Exception {
Connection conn = cp.getConnection();
try {
+
PreparedStatement prep = null;
PreparedStatement prepData = null;
+
+ StringBuilder inClause = new StringBuilder();
+ int batch = chunkIds.size();
+ for (int i = 0; i < batch; i++) {
+ inClause.append('?');
+ if (i != batch -1) {
+ inClause.append(',');
+ }
+ }
if (maxLastModifiedTime > 0) {
prep = conn.prepareStatement(
- "delete from datastore_meta where id = ? and lastMod
<= ?");
- prep.setLong(2, maxLastModifiedTime);
+ "delete from datastore_meta where id in ("
+ + inClause.toString() + ") and lastMod <= ?");
+ prep.setLong(batch + 1, maxLastModifiedTime);
prepData = conn.prepareStatement(
- "delete from datastore_data where id = ? and lastMod
<= ?");
- prepData.setLong(2, maxLastModifiedTime);
+ "delete from datastore_data where id in ("
+ + inClause.toString() + ") and lastMod <= ?");
+ prepData.setLong(batch + 1, maxLastModifiedTime);
} else {
prep = conn.prepareStatement(
- "delete from datastore_meta where id = ?");
+ "delete from datastore_meta where id in ("
+ + inClause.toString() + ")");
prepData = conn.prepareStatement(
- "delete from datastore_data where id = ?");
+ "delete from datastore_data where id in ("
+ + inClause.toString() + ")");
+ }
+
+ for (int idx = 0; idx < batch; idx++) {
+ prep.setString(idx + 1, chunkIds.get(idx));
+ prepData.setString(idx + 1, chunkIds.get(idx));
}
- prep.setString(1, chunkId);
+
prep.execute();
- prepData.setString(1, chunkId);
prepData.execute();
-
prep.close();
prepData.close();
} finally {
@@ -256,6 +274,7 @@ public class DbBlobStore extends Caching
return true;
}
+
@Override
public Iterator<String> getAllChunkIds(long maxLastModifiedTime) throws
Exception {
final Connection conn = cp.getConnection();
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java?rev=1579175&r1=1579174&r2=1579175&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
Wed Mar 19 08:55:44 2014
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.plugin
import java.io.IOException;
import java.util.Iterator;
+import java.util.List;
import org.apache.jackrabbit.oak.commons.StringUtils;
import org.apache.jackrabbit.oak.plugins.blob.CachingBlobStore;
@@ -195,11 +196,19 @@ public class MongoBlobStore extends Cach
}
@Override
- public boolean deleteChunk(String chunkId, long maxLastModifiedTime)
throws Exception {
+ public boolean deleteChunks(List<String> chunkIds, long
maxLastModifiedTime) throws Exception {
DBCollection collection = getBlobCollection();
- WriteResult result = collection.remove(getBlobQuery(chunkId,
maxLastModifiedTime));
+ QueryBuilder queryBuilder = new QueryBuilder();
+ if (chunkIds != null) {
+ queryBuilder =
queryBuilder.and(MongoBlob.KEY_ID).in(chunkIds.toArray(new String[0]));
+ if (maxLastModifiedTime > 0) {
+ queryBuilder = queryBuilder.and(MongoBlob.KEY_LAST_MOD)
+ .lessThan(maxLastModifiedTime);
+ }
+ }
- if (result.getN() == 1) {
+ WriteResult result = collection.remove(queryBuilder.get());
+ if (result.getN() == chunkIds.size()) {
return true;
}
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java?rev=1579175&r1=1579174&r2=1579175&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBBlobStore.java
Wed Mar 19 08:55:44 2014
@@ -25,6 +25,7 @@ import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import javax.sql.DataSource;
@@ -288,30 +289,47 @@ public class RDBBlobStore extends Cachin
}
@Override
- public boolean deleteChunk(String chunkId, long maxLastModifiedTime)
throws Exception {
+ public boolean deleteChunks(List<String> chunkIds, long
maxLastModifiedTime) throws Exception {
try {
PreparedStatement prep = null;
PreparedStatement prepData = null;
+ StringBuilder inClause = new StringBuilder();
+ int batch = chunkIds.size();
+ for (int i = 0; i < batch; i++) {
+ inClause.append('?');
+ if (i != batch - 1) {
+ inClause.append(',');
+ }
+ }
+
if (maxLastModifiedTime > 0) {
prep = connection.prepareStatement(
- "delete from datastore_meta where id = ? and lastMod
<= ?");
- prep.setLong(2, maxLastModifiedTime);
+ "delete from datastore_meta where id in ("
+ + inClause.toString() + ") and lastMod <= ?");
+ prep.setLong(batch + 1, maxLastModifiedTime);
prepData = connection.prepareStatement(
- "delete from datastore_data where id = ? and lastMod
<= ?");
- prepData.setLong(2, maxLastModifiedTime);
+ "delete from datastore_data where id in ("
+ + inClause.toString() + ") and lastMod <= ?");
+ prepData.setLong(batch + 1, maxLastModifiedTime);
} else {
prep = connection.prepareStatement(
- "delete from datastore_meta where id = ?");
+ "delete from datastore_meta where id in ("
+ + inClause.toString() + ")");
+
prepData = connection.prepareStatement(
- "delete from datastore_data where id = ?");
+ "delete from datastore_data where id in ("
+ + inClause.toString() + ")");
}
- prep.setString(1, chunkId);
+
+ for (int idx = 0; idx < batch; idx++) {
+ prep.setString(idx + 1, chunkIds.get(idx));
+ prepData.setString(idx + 1, chunkIds.get(idx));
+ }
+
prep.execute();
- prepData.setString(1, chunkId);
prepData.execute();
-
prep.close();
prepData.close();
} finally {
