Author: reschke
Date: Tue Sep 5 14:02:15 2017
New Revision: 1807352
URL: http://svn.apache.org/viewvc?rev=1807352&view=rev
Log:
OAK-6162: BlobReferenceIterator refactoring (ported to 1.6)
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 5 14:02:15 2017
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785917,1785919,1785946,1786122,1787074,1787145,1787217,1787425,1788056,1788378,1788387-1788389,1788850,1789056,1789534,1790382,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1798035,1798834,1799219,1799389,1799393,1799924,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119,1801675,1802260,1802262,1802286,1802548,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1805851-1805852,1806668
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785917,1785919,1785946,1786122,1787074,1787145,1787217,1787425,1788056,1788378,1788387-1788389,1788850,1789056,1789534,1790382,1792463,1792742,1792746,1793013,1793088,1793618,1793627,1793644,1794393,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618,1796144,1796230,1796239,1796274,1796278,1796988,1798035,1798834,1799219,1799389,1799393,1799924,1800269,1800606,1800613,1800974,1801011,1801013,1801118-1801119,1801675,1802260,1802262,1802286,1802548,1802934,1802938,1802973,1803026,1803247-1803249,1803951,1803953-1803955,1805851-1805852,1806668
/jackrabbit/trunk:1345480
Modified:
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java?rev=1807352&r1=1807351&r2=1807352&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java
Tue Sep 5 14:02:15 2017
@@ -16,96 +16,76 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
-import java.util.HashSet;
+import java.io.Closeable;
import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
+import java.util.Queue;
import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+
+import com.google.common.collect.AbstractIterator;
+import com.google.common.collect.Queues;
/**
* An iterator over all referenced binaries.
* <p>
- * Only top-level referenced are returned (indirection, if any, is not
resolved).
- * The items are returned in no particular order.
- * An item might be returned multiple times.
+ * Only top-level referenced are returned (indirection, if any, is not
+ * resolved). The items are returned in no particular order. An item might be
+ * returned multiple times.
*/
-public class BlobReferenceIterator implements Iterator<ReferencedBlob> {
+public class BlobReferenceIterator extends AbstractIterator<ReferencedBlob>
implements Closeable {
- private static final int BATCH_SIZE = 1000;
- private final DocumentStore docStore;
+ private final DocumentStore documentStore;
private final BlobCollector blobCollector;
- private HashSet<ReferencedBlob> batch = new HashSet<ReferencedBlob>();
- private Iterator<ReferencedBlob> batchIterator;
- private boolean done;
- private String fromKey = NodeDocument.MIN_ID_VALUE;
+ private final Queue<ReferencedBlob> blobs = Queues.newArrayDeque();
+
+ private Iterator<NodeDocument> iterator;
public BlobReferenceIterator(DocumentNodeStore nodeStore) {
- this.docStore = nodeStore.getDocumentStore();
- batchIterator = batch.iterator();
+ this.documentStore = nodeStore.getDocumentStore();
this.blobCollector = new BlobCollector(nodeStore);
}
@Override
- public boolean hasNext() {
- if (!batchIterator.hasNext()) {
+ protected ReferencedBlob computeNext() {
+ if (blobs.isEmpty()) {
loadBatch();
}
- return batchIterator.hasNext() || !done;
- }
- @Override
- public ReferencedBlob next() {
- // this will load the next batch if required
- if (!hasNext()) {
- throw new NoSuchElementException();
+ if (!blobs.isEmpty()) {
+ return blobs.remove();
+ } else {
+ return endOfData();
}
- return batchIterator.next();
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
}
private void loadBatch() {
- if (done) {
- return;
+ if (this.iterator == null) {
+ this.iterator = getIteratorOverDocsWithBinaries();
}
- batch.clear();
- // read until at least BATCH_SIZE references are available
- while (true) {
- boolean hasMore = loadBatchQuery();
- if (!hasMore) {
- done = true;
- break;
- }
- if (batch.size() > BATCH_SIZE) {
- break;
- }
+ // Some node which have the '_bin' flag set might not have any binaries
+ // in it so move forward if blobs is still empty and cursor has more
+ // elements
+ while (iterator.hasNext() && blobs.isEmpty()) {
+ collectBinaries(iterator.next());
}
- batchIterator = batch.iterator();
}
- private boolean loadBatchQuery() {
- // read about BATCH_SIZE documents
- List<NodeDocument> list =
- docStore.query(Collection.NODES, fromKey,
NodeDocument.MAX_ID_VALUE, NodeDocument.HAS_BINARY_FLAG,
- NodeDocument.HAS_BINARY_VAL,
- BATCH_SIZE);
- boolean hasMore = false;
- for (NodeDocument doc : list) {
- if (doc.getId().equals(fromKey)) {
- // already read
- continue;
- }
- hasMore = true;
- fromKey = doc.getId();
- blobCollector.collect(doc, batch);
- }
- return hasMore;
+ private void collectBinaries(NodeDocument nodeDocument) {
+ blobCollector.collect(nodeDocument, blobs);
}
+ /**
+ * Override this document to use a document store specific iterator.
+ */
+ public Iterator<NodeDocument> getIteratorOverDocsWithBinaries() {
+ int batchSize = 1000;
+ return Utils.getSelectedDocuments(documentStore,
NodeDocument.HAS_BINARY_FLAG, NodeDocument.HAS_BINARY_VAL, batchSize)
+ .iterator();
+ }
-
+ @Override
+ public void close() {
+ Utils.closeIfCloseable(iterator);
+ }
}
Modified:
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java?rev=1807352&r1=1807351&r2=1807352&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java
Tue Sep 5 14:02:15 2017
@@ -18,15 +18,14 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
-import java.io.Closeable;
import java.util.Iterator;
import org.apache.jackrabbit.oak.api.Blob;
-import org.apache.jackrabbit.oak.commons.IOUtils;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,30 +43,29 @@ public class DocumentBlobReferenceRetrie
@Override
public void collectReferences(ReferenceCollector collector) {
int referencesFound = 0;
- Iterator<ReferencedBlob> blobIterator =
nodeStore.getReferencedBlobsIterator();
+ Iterator<ReferencedBlob> blobIterator = null;
try {
+ blobIterator = nodeStore.getReferencedBlobsIterator();
while (blobIterator.hasNext()) {
ReferencedBlob refBlob = blobIterator.next();
Blob blob = refBlob.getBlob();
referencesFound++;
- //TODO this mode would also add in memory blobId
- //Would that be an issue
+ // TODO this mode would also add in memory blobId
+ // Would that be an issue
if (blob instanceof BlobStoreBlob) {
collector.addReference(((BlobStoreBlob) blob).getBlobId(),
refBlob.getId());
} else {
- //TODO Should not rely on toString. Instead obtain
- //secure reference and convert that to blobId using
- //blobStore
+ // TODO Should not rely on toString. Instead obtain
+ // secure reference and convert that to blobId using
+ // blobStore
collector.addReference(blob.toString(), refBlob.getId());
}
}
- }finally{
- if(blobIterator instanceof Closeable){
- IOUtils.closeQuietly((Closeable) blobIterator);
- }
+ } finally {
+ Utils.closeIfCloseable(blobIterator);
}
log.debug("Total blob references found (including chunk resolution)
[{}]", referencesFound);
}
Modified:
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java?rev=1807352&r1=1807351&r2=1807352&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
(original)
+++
jackrabbit/oak/branches/1.6/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java
Tue Sep 5 14:02:15 2017
@@ -567,6 +567,9 @@ public class Utils {
return c.compare(a, b) <= 0 ? a : b;
}
+ // default batch size for paging through a document store
+ private static final int DEFAULT_BATCH_SIZE = 100;
+
/**
* Returns an {@link Iterable} over all {@link NodeDocument}s in the given
* store. The returned {@linkplain Iterable} does not guarantee a
consistent
@@ -578,7 +581,7 @@ public class Utils {
* @return an {@link Iterable} over all documents in the store.
*/
public static Iterable<NodeDocument> getAllDocuments(final DocumentStore
store) {
- return internalGetSelectedDocuments(store, null, 0);
+ return internalGetSelectedDocuments(store, null, 0,
DEFAULT_BATCH_SIZE);
}
/**
@@ -614,23 +617,35 @@ public class Utils {
* @param indexedProperty the name of the indexed property.
* @param startValue the lower bound value for the indexed property
* (inclusive).
+ * @param batchSize number of documents to fetch at once
* @return an {@link Iterable} over all documents in the store matching the
* condition
*/
public static Iterable<NodeDocument> getSelectedDocuments(
+ DocumentStore store, String indexedProperty, long startValue, int
batchSize) {
+ return internalGetSelectedDocuments(store, indexedProperty,
startValue, batchSize);
+ }
+
+ /**
+ * Like {@link #getSelectedDocuments(DocumentStore, String, long, int)}
with
+ * a default {@code batchSize}.
+ */
+ public static Iterable<NodeDocument> getSelectedDocuments(
DocumentStore store, String indexedProperty, long startValue) {
- return internalGetSelectedDocuments(store, indexedProperty,
startValue);
+ return internalGetSelectedDocuments(store, indexedProperty,
startValue, DEFAULT_BATCH_SIZE);
}
private static Iterable<NodeDocument> internalGetSelectedDocuments(
final DocumentStore store, final String indexedProperty,
- final long startValue) {
+ final long startValue, final int batchSize) {
+ if (batchSize < 2) {
+ throw new IllegalArgumentException("batchSize must be > 1");
+ }
return new Iterable<NodeDocument>() {
@Override
public Iterator<NodeDocument> iterator() {
return new AbstractIterator<NodeDocument>() {
- private static final int BATCH_SIZE = 100;
private String startId = NodeDocument.MIN_ID_VALUE;
private Iterator<NodeDocument> batch = nextBatch();
@@ -655,8 +670,8 @@ public class Utils {
private Iterator<NodeDocument> nextBatch() {
List<NodeDocument> result = indexedProperty == null ?
store.query(Collection.NODES, startId,
- NodeDocument.MAX_ID_VALUE, BATCH_SIZE) :
store.query(Collection.NODES, startId,
- NodeDocument.MAX_ID_VALUE, indexedProperty,
startValue, BATCH_SIZE);
+ NodeDocument.MAX_ID_VALUE, batchSize) :
store.query(Collection.NODES, startId,
+ NodeDocument.MAX_ID_VALUE, indexedProperty,
startValue, batchSize);
return result.iterator();
}
};
