Author: chetanm
Date: Fri Sep 15 05:49:00 2017
New Revision: 1808411
URL: http://svn.apache.org/viewvc?rev=1808411&view=rev
Log:
OAK-6353 - Use Document order traversal for reindexing performed on
DocumentNodeStore setups
Refactor logic related to read all NodeState as NodeStateEntryTraverser
which provides an iterable access to all NodeState entries
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntryTraverser.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java?rev=1808411&r1=1808410&r2=1808411&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
Fri Sep 15 05:49:00 2017
@@ -26,21 +26,15 @@ import java.util.List;
import java.util.Set;
import com.codahale.metrics.MetricRegistry;
-import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.io.Closer;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.index.IndexHelper;
import org.apache.jackrabbit.oak.index.IndexerSupport;
-import org.apache.jackrabbit.oak.plugins.document.Collection;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeState;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
-import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
-import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentTraverser;
-import org.apache.jackrabbit.oak.plugins.document.util.CloseableIterable;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
-import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
import org.apache.jackrabbit.oak.plugins.index.NodeTraversalCallback;
import
org.apache.jackrabbit.oak.plugins.index.progress.IndexingProgressReporter;
@@ -96,24 +90,19 @@ public class DocumentStoreIndexer implem
DocumentNodeStore nodeStore = (DocumentNodeStore)
indexHelper.getNodeStore();
progressReporter.reindexingTraversalStart("/");
- for (NodeDocument doc : getIncludedDocs(indexer)) {
- String path = doc.getPath();
- DocumentNodeState nodeState = nodeStore.getNode(path,
rootDocumentState.getRootRevision());
+ NodeStateEntryTraverser nsep =
+ new
NodeStateEntryTraverser(rootDocumentState.getRootRevision(),
+ nodeStore, getMongoDocumentStore())
+ .withProgressCallback(this::reportDocumentRead)
+ .withPathPredicate(indexer::shouldInclude);
- //At DocumentNodeState api level the nodeState can be null
- if (nodeState == null || !nodeState.exists()) {
- continue;
- }
-
- NodeStateEntry entry = new NodeStateEntry(nodeState, path);
+ for (NodeStateEntry entry : nsep) {
indexer.index(entry);
-
- for (DocumentNodeState bundledState :
nodeState.getAllBundledNodesStates()) {
- indexer.index(new NodeStateEntry(bundledState,
bundledState.getPath()));
- }
}
+ nsep.close();
+
progressReporter.reindexingTraversalEnd();
progressReporter.logReport();
@@ -122,6 +111,10 @@ public class DocumentStoreIndexer implem
indexerSupport.postIndexWork(copyOnWriteStore);
}
+ private MongoDocumentStore getMongoDocumentStore() {
+ return checkNotNull(indexHelper.getService(MongoDocumentStore.class));
+ }
+
private void configureEstimators() {
StatisticsProvider statsProvider = indexHelper.getStatisticsProvider();
if (statsProvider instanceof MetricStatisticsProvider) {
@@ -142,50 +135,6 @@ public class DocumentStoreIndexer implem
closer.close();
}
- @SuppressWarnings("Guava")
- private Iterable<NodeDocument> getIncludedDocs(CompositeIndexer indexer) {
- return FluentIterable.from(getDocsFilteredByPath(indexer))
- .filter(doc -> !doc.isSplitDocument())
- .filter(indexer::shouldInclude);
- }
-
- private Iterable<NodeDocument> getDocsFilteredByPath(CompositeIndexer
indexer) {
- CloseableIterable<NodeDocument> docs = findAllDocuments(indexer);
- closer.register(docs);
- return docs;
- }
-
- private CloseableIterable<NodeDocument> findAllDocuments(CompositeIndexer
indexer) {
- MongoDocumentStore mds =
indexHelper.getService(MongoDocumentStore.class);
- checkNotNull(mds);
- return new
MongoDocumentTraverser(mds).getAllDocuments(Collection.NODES, id ->
includeId(id, indexer));
- }
-
- private boolean includeId(String id, NodeStateIndexer indexer) {
- reportDocumentRead(id);
- //Cannot interpret long paths as they are hashed. So let them
- //be included
- if (Utils.isIdFromLongPath(id)){
- return true;
- }
-
- //Not easy to determine path for previous docs
- //Given there count is pretty low compared to others
- //include them all
- if (Utils.isPreviousDocId(id)){
- return true;
- }
-
- String path = Utils.getPathFromId(id);
-
- //Exclude hidden nodes from index data
- if (NodeStateUtils.isHiddenPath(path)){
- return false;
- }
-
- return indexer.shouldInclude(path);
- }
-
private void reportDocumentRead(String id) {
try {
progressReporter.traversedNode(() -> id);
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntryTraverser.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntryTraverser.java?rev=1808411&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntryTraverser.java
(added)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntryTraverser.java
Fri Sep 15 05:49:00 2017
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index.indexer.document;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+
+import com.google.common.collect.FluentIterable;
+import com.google.common.io.Closer;
+import org.apache.jackrabbit.oak.plugins.document.Collection;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeState;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
+import org.apache.jackrabbit.oak.plugins.document.RevisionVector;
+import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
+import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentTraverser;
+import org.apache.jackrabbit.oak.plugins.document.util.CloseableIterable;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.jetbrains.annotations.NotNull;
+
+import static com.google.common.collect.Iterables.concat;
+import static com.google.common.collect.Iterables.transform;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singleton;
+
+public class NodeStateEntryTraverser implements Iterable<NodeStateEntry>,
Closeable {
+ private final Closer closer = Closer.create();
+ private final RevisionVector rootRevision;
+ private final DocumentNodeStore documentNodeStore;
+ private final MongoDocumentStore documentStore;
+
+ private Consumer<String> progressReporter = id -> {};
+ private Predicate<String> pathPredicate = path -> true;
+
+ public NodeStateEntryTraverser(DocumentNodeStore documentNodeStore,
+ MongoDocumentStore documentStore) {
+ this(documentNodeStore.getHeadRevision(), documentNodeStore,
documentStore);
+ }
+
+ public NodeStateEntryTraverser(RevisionVector rootRevision,
DocumentNodeStore documentNodeStore,
+ MongoDocumentStore documentStore) {
+ this.rootRevision = rootRevision;
+ this.documentNodeStore = documentNodeStore;
+ this.documentStore = documentStore;
+ }
+
+ @NotNull
+ @Override
+ public Iterator<NodeStateEntry> iterator() {
+ return getIncludedDocs().iterator();
+ }
+
+ public NodeStateEntryTraverser withProgressCallback(Consumer<String>
progressReporter) {
+ this.progressReporter = progressReporter;
+ return this;
+ }
+
+ public NodeStateEntryTraverser withPathPredicate(Predicate<String>
pathPredicate) {
+ this.pathPredicate = pathPredicate;
+ return this;
+ }
+
+ @Override
+ public void close() throws IOException {
+ closer.close();
+ }
+
+ @SuppressWarnings("Guava")
+ private Iterable<NodeStateEntry> getIncludedDocs() {
+ return FluentIterable.from(getDocsFilteredByPath())
+ .filter(doc -> !doc.isSplitDocument())
+ .transformAndConcat(doc -> getEntries(doc));
+ }
+
+ @SuppressWarnings("StaticPseudoFunctionalStyleMethod")
+ private Iterable<NodeStateEntry> getEntries(NodeDocument doc) {
+ String path = doc.getPath();
+
+ DocumentNodeState nodeState = documentNodeStore.getNode(path,
rootRevision);
+
+ //At DocumentNodeState api level the nodeState can be null
+ if (nodeState == null || !nodeState.exists()) {
+ return emptyList();
+ }
+
+ return transform(
+ concat(singleton(nodeState),
+ nodeState.getAllBundledNodesStates()),
+ dns -> new NodeStateEntry(dns, dns.getPath())
+ );
+ }
+
+ private Iterable<NodeDocument> getDocsFilteredByPath() {
+ CloseableIterable<NodeDocument> docs = findAllDocuments();
+ closer.register(docs);
+ return docs;
+ }
+
+ private CloseableIterable<NodeDocument> findAllDocuments() {
+ return new MongoDocumentTraverser(documentStore)
+ .getAllDocuments(Collection.NODES, id -> includeId(id));
+ }
+
+ private boolean includeId(String id) {
+ progressReporter.accept(id);
+ //Cannot interpret long paths as they are hashed. So let them
+ //be included
+ if (Utils.isIdFromLongPath(id)){
+ return true;
+ }
+
+ //Not easy to determine path for previous docs
+ //Given there count is pretty low compared to others
+ //include them all so that they become part of cache
+ if (Utils.isPreviousDocId(id)){
+ return true;
+ }
+
+ String path = Utils.getPathFromId(id);
+
+ //Exclude hidden nodes from index data
+ if (NodeStateUtils.isHiddenPath(path)){
+ return false;
+ }
+
+ return pathPredicate.test(path);
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateEntryTraverser.java
------------------------------------------------------------------------------
svn:eol-style = native
