Author: chetanm
Date: Wed Sep 6 09:03:27 2017
New Revision: 1807452
URL: http://svn.apache.org/viewvc?rev=1807452&view=rev
Log:
OAK-6353 - Use Document order traversal for reindexing performed on
DocumentNodeStore setups
Complete the support for adding new index definition
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/LuceneIndexerProvider.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateIndexerProvider.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java?rev=1807452&r1=1807451&r2=1807452&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexerSupport.java
Wed Sep 6 09:03:27 2017
@@ -27,9 +27,14 @@ import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.felix.inventory.Format;
import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.importer.AsyncLaneSwitcher;
import org.apache.jackrabbit.oak.plugins.index.importer.IndexDefinitionUpdater;
import org.apache.jackrabbit.oak.plugins.index.importer.IndexerInfo;
import
org.apache.jackrabbit.oak.plugins.index.inventory.IndexDefinitionPrinter;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
@@ -37,6 +42,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
public class IndexerSupport {
private final Logger log = LoggerFactory.getLogger(getClass());
@@ -50,6 +56,11 @@ public class IndexerSupport {
* This would be mostly used for testing purpose
*/
private static final String HEAD_AS_CHECKPOINT = "head";
+
+ /**
+ * Index lane name which is used for indexing
+ */
+ private static final String REINDEX_LANE = "offline-reindex-async";
private Map<String, String> checkpointInfo = Collections.emptyMap();
private final IndexHelper indexHelper;
private File localIndexDir;
@@ -92,13 +103,13 @@ public class IndexerSupport {
return checkpointedState;
}
- public void updateIndexDefinitions(NodeBuilder rootBuilder) throws
IOException, CommitFailedException {
+ private void updateIndexDefinitions(NodeBuilder rootBuilder) throws
IOException, CommitFailedException {
if (indexDefinitions != null) {
new IndexDefinitionUpdater(indexDefinitions).apply(rootBuilder);
}
}
- public void dumpIndexDefinitions(NodeStore nodeStore) throws IOException,
CommitFailedException {
+ private void dumpIndexDefinitions(NodeStore nodeStore) throws IOException,
CommitFailedException {
IndexDefinitionPrinter printer = new IndexDefinitionPrinter(nodeStore,
indexHelper.getIndexPathService());
printer.setFilter("{\"properties\":[\"*\",
\"-:childOrder\"],\"nodes\":[\"*\", \"-:index-definition\"]}");
PrinterDumper dumper = new PrinterDumper(getLocalIndexDir(),
IndexDefinitionUpdater.INDEX_DEFINITIONS_JSON,
@@ -106,6 +117,42 @@ public class IndexerSupport {
dumper.dump();
}
+ public void switchIndexLanesAndReindexFlag(NodeStore copyOnWriteStore)
throws CommitFailedException, IOException {
+ NodeState root = copyOnWriteStore.getRoot();
+ NodeBuilder builder = root.builder();
+ updateIndexDefinitions(builder);
+
+ for (String indexPath : indexHelper.getIndexPaths()) {
+ //TODO Do it only for lucene indexes for now
+ NodeBuilder idxBuilder = childBuilder(builder, indexPath, false);
+ checkState(idxBuilder.exists(), "No index definition found at path
[%s]", indexPath);
+
+ idxBuilder.setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true);
+ AsyncLaneSwitcher.switchLane(idxBuilder, REINDEX_LANE);
+ }
+
+ copyOnWriteStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ log.info("Switched the async lane for indexes at {} to {} and marked
them for reindex", indexHelper.getIndexPaths(), REINDEX_LANE);
+ }
+
+ public void postIndexWork(NodeStore copyOnWriteStore) throws
CommitFailedException, IOException {
+ switchIndexLanesBack(copyOnWriteStore);
+ dumpIndexDefinitions(copyOnWriteStore);
+ }
+
+ private void switchIndexLanesBack(NodeStore copyOnWriteStore) throws
CommitFailedException, IOException {
+ NodeState root = copyOnWriteStore.getRoot();
+ NodeBuilder builder = root.builder();
+
+ for (String indexPath : indexHelper.getIndexPaths()) {
+ NodeBuilder idxBuilder = childBuilder(builder, indexPath, false);
+ AsyncLaneSwitcher.revertSwitch(idxBuilder, indexPath);
+ }
+
+ copyOnWriteStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ log.info("Switched the async lane for indexes at {} back to there
original lanes", indexHelper.getIndexPaths());
+ }
+
public Map<String, String> getCheckpointInfo() {
return checkpointInfo;
}
@@ -113,4 +160,11 @@ public class IndexerSupport {
public void setIndexDefinitions(File indexDefinitions) {
this.indexDefinitions = indexDefinitions;
}
+
+ public static NodeBuilder childBuilder(NodeBuilder nb, String path,
boolean createNew) {
+ for (String name : PathUtils.elements(checkNotNull(path))) {
+ nb = createNew ? nb.child(name) : nb.getChildNode(name);
+ }
+ return nb;
+ }
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java?rev=1807452&r1=1807451&r2=1807452&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
Wed Sep 6 09:03:27 2017
@@ -89,10 +89,9 @@ public class OutOfBandIndexer implements
NodeState baseState = copyOnWriteStore.getRoot();
//TODO Check for indexPaths being empty
- switchIndexLanesAndReindexFlag();
+ indexerSupport.switchIndexLanesAndReindexFlag(copyOnWriteStore);
preformIndexUpdate(baseState);
- switchIndexLanesBack();
- indexerSupport.dumpIndexDefinitions(copyOnWriteStore);
+ indexerSupport.postIndexWork(copyOnWriteStore);
}
private File getLocalIndexDir() throws IOException {
@@ -163,37 +162,6 @@ public class OutOfBandIndexer implements
return luceneIndexHelper.createEditorProvider();
}
- private void switchIndexLanesAndReindexFlag() throws
CommitFailedException, IOException {
- NodeState root = copyOnWriteStore.getRoot();
- NodeBuilder builder = root.builder();
- indexerSupport.updateIndexDefinitions(builder);
-
- for (String indexPath : indexHelper.getIndexPaths()) {
- //TODO Do it only for lucene indexes for now
- NodeBuilder idxBuilder = childBuilder(builder, indexPath, false);
- checkState(idxBuilder.exists(), "No index definition found at path
[%s]", indexPath);
-
- idxBuilder.setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true);
- AsyncLaneSwitcher.switchLane(idxBuilder, REINDEX_LANE);
- }
-
- copyOnWriteStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
- log.info("Switched the async lane for indexes at {} to {} and marked
them for reindex", indexHelper.getIndexPaths(), REINDEX_LANE);
- }
-
- private void switchIndexLanesBack() throws CommitFailedException,
IOException {
- NodeState root = copyOnWriteStore.getRoot();
- NodeBuilder builder = root.builder();
-
- for (String indexPath : indexHelper.getIndexPaths()) {
- NodeBuilder idxBuilder = childBuilder(builder, indexPath, false);
- AsyncLaneSwitcher.revertSwitch(idxBuilder, indexPath);
- }
-
- copyOnWriteStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
- log.info("Switched the async lane for indexes at {} back to there
original lanes", indexHelper.getIndexPaths());
- }
-
private void configureEstimators(IndexUpdate indexUpdate) {
StatisticsProvider statsProvider = indexHelper.getStatisticsProvider();
if (statsProvider instanceof MetricStatisticsProvider) {
@@ -205,10 +173,4 @@ public class OutOfBandIndexer implements
indexUpdate.setNodeCountEstimator(estimator);
}
- private static NodeBuilder childBuilder(NodeBuilder nb, String path,
boolean createNew) {
- for (String name : PathUtils.elements(checkNotNull(path))) {
- nb = createNew ? nb.child(name) : nb.getChildNode(name);
- }
- return nb;
- }
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java?rev=1807452&r1=1807451&r2=1807452&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
Wed Sep 6 09:03:27 2017
@@ -45,9 +45,14 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.NodeTraversalCallback;
import
org.apache.jackrabbit.oak.plugins.index.progress.IndexingProgressReporter;
import org.apache.jackrabbit.oak.plugins.index.progress.MetricRateEstimator;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
import org.apache.jackrabbit.oak.plugins.metric.MetricStatisticsProvider;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,8 +78,13 @@ public class DocumentStoreIndexer implem
public void reindex() throws CommitFailedException, IOException {
configureEstimators();
- //TODO Support for adding new index definition
- CompositeIndexer indexer = prepareIndexers();
+
+ NodeState checkpointedState =
indexerSupport.retrieveNodeStateForCheckpoint();
+ NodeStore copyOnWriteStore = new MemoryNodeStore(checkpointedState);
+ indexerSupport.switchIndexLanesAndReindexFlag(copyOnWriteStore);
+
+ NodeBuilder builder = copyOnWriteStore.getRoot().builder();
+ CompositeIndexer indexer = prepareIndexers(copyOnWriteStore, builder);
if (indexer.isEmpty()) {
return;
}
@@ -82,7 +92,6 @@ public class DocumentStoreIndexer implem
closer.register(indexer);
//TODO How to ensure we can safely read from secondary
- NodeState checkpointedState =
indexerSupport.retrieveNodeStateForCheckpoint();
DocumentNodeState rootDocumentState = (DocumentNodeState)
checkpointedState;
DocumentNodeStore nodeStore = (DocumentNodeStore)
indexHelper.getNodeStore();
@@ -107,6 +116,10 @@ public class DocumentStoreIndexer implem
progressReporter.reindexingTraversalEnd();
progressReporter.logReport();
+
+ copyOnWriteStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ indexerSupport.postIndexWork(copyOnWriteStore);
}
private void configureEstimators() {
@@ -182,18 +195,19 @@ public class DocumentStoreIndexer implem
traversalLog.trace(id);
}
- protected CompositeIndexer prepareIndexers() {
- NodeState root = indexHelper.getNodeStore().getRoot();
+ protected CompositeIndexer prepareIndexers(NodeStore copyOnWriteStore,
NodeBuilder builder) {
+ NodeState root = copyOnWriteStore.getRoot();
List<NodeStateIndexer> indexers = new ArrayList<>();
for (String indexPath : indexHelper.getIndexPaths()) {
NodeState indexState = NodeStateUtils.getNode(root, indexPath);
+ NodeBuilder idxBuilder = IndexerSupport.childBuilder(builder,
indexPath, false);
String type = indexState.getString(TYPE_PROPERTY_NAME);
if (type == null) {
log.warn("No 'type' property found on indexPath [{}]. Skipping
it", indexPath);
continue;
}
for (NodeStateIndexerProvider indexerProvider : indexerProviders) {
- NodeStateIndexer indexer = indexerProvider.getIndexer(type,
indexPath, indexState, root, progressReporter);
+ NodeStateIndexer indexer = indexerProvider.getIndexer(type,
indexPath, idxBuilder, root, progressReporter);
if (indexer != null) {
indexers.add(indexer);
closer.register(indexer);
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/LuceneIndexerProvider.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/LuceneIndexerProvider.java?rev=1807452&r1=1807451&r2=1807452&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/LuceneIndexerProvider.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/LuceneIndexerProvider.java
Wed Sep 6 09:03:27 2017
@@ -57,21 +57,20 @@ public class LuceneIndexerProvider imple
@Override
public NodeStateIndexer getIndexer(@Nonnull String type, @Nonnull String
indexPath,
- @Nonnull NodeState definition, @Nonnull
NodeState root,
+ @Nonnull NodeBuilder definition,
@Nonnull NodeState root,
IndexingProgressReporter
progressReporter) {
if (!TYPE_LUCENE.equals(definition.getString(TYPE_PROPERTY_NAME))) {
return null;
}
- NodeBuilder builder = definition.builder();
- IndexDefinition idxDefinition = IndexDefinition.newBuilder(root,
definition, indexPath).reindex().build();
+ IndexDefinition idxDefinition = IndexDefinition.newBuilder(root,
definition.getNodeState(), indexPath).reindex().build();
- LuceneIndexWriter indexWriter =
indexWriterFactory.newInstance(idxDefinition, builder, true);
+ LuceneIndexWriter indexWriter =
indexWriterFactory.newInstance(idxDefinition, definition, true);
BinaryTextExtractor textExtractor = new BinaryTextExtractor(textCache,
idxDefinition, true);
return new LuceneIndexer(
idxDefinition,
indexWriter,
- builder,
+ definition,
textExtractor,
progressReporter
);
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateIndexerProvider.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateIndexerProvider.java?rev=1807452&r1=1807451&r2=1807452&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateIndexerProvider.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/NodeStateIndexerProvider.java
Wed Sep 6 09:03:27 2017
@@ -25,6 +25,7 @@ import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import
org.apache.jackrabbit.oak.plugins.index.progress.IndexingProgressReporter;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
public interface NodeStateIndexerProvider extends Closeable {
@@ -32,6 +33,6 @@ public interface NodeStateIndexerProvide
@CheckForNull
NodeStateIndexer getIndexer(@Nonnull String type,
@Nonnull String indexPath,
- @Nonnull NodeState definition,
+ @Nonnull NodeBuilder definition,
@Nonnull NodeState root,
IndexingProgressReporter progressReporter);
}
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java?rev=1807452&r1=1807451&r2=1807452&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
Wed Sep 6 09:03:27 2017
@@ -46,10 +46,12 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexRootDirectory;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.LocalIndexDir;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.spi.whiteboard.DefaultWhiteboard;
import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
@@ -61,7 +63,9 @@ import org.junit.Test;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyMap;
import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE;
+import static
org.apache.jackrabbit.oak.plugins.document.TestUtils.childBuilder;
import static org.apache.jackrabbit.oak.plugins.document.TestUtils.createChild;
+import static org.apache.jackrabbit.oak.plugins.document.TestUtils.merge;
import static
org.apache.jackrabbit.oak.plugins.document.bundlor.BundlingConfigHandler.BUNDLOR;
import static
org.apache.jackrabbit.oak.plugins.document.bundlor.BundlingConfigHandler.DOCUMENT_NODE_STORE;
import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
@@ -129,7 +133,8 @@ public class DocumentStoreIndexerIT exte
wb.register(MongoDocumentStore.class, ds, emptyMap());
wb.register(StatisticsProvider.class, StatisticsProvider.NOOP,
emptyMap());
- configureBunding(store);
+ configureIndex(store);
+ configureBundling(store);
NodeBuilder builder = store.getRoot().builder();
NodeBuilder appNB = newNode("app:Asset");
@@ -152,13 +157,13 @@ public class DocumentStoreIndexerIT exte
String checkpoint = store.checkpoint(100000);
IndexHelper helper = new IndexHelper(store, store.getBlobStore(), wb,
temporaryFolder.newFolder(),
- temporaryFolder.newFolder(), asList("/oak:index/foo"));
+ temporaryFolder.newFolder(), asList(TEST_INDEX_PATH));
IndexerSupport support = new IndexerSupport(helper, checkpoint);
CollectingIndexer testIndexer = new CollectingIndexer(p ->
p.startsWith("/test"));
DocumentStoreIndexer index = new DocumentStoreIndexer(helper, support)
{
@Override
- protected CompositeIndexer prepareIndexers() {
+ protected CompositeIndexer prepareIndexers(NodeStore nodeStore,
NodeBuilder builder) {
return new CompositeIndexer(asList(testIndexer));
}
};
@@ -182,6 +187,17 @@ public class DocumentStoreIndexerIT exte
}
+ private void configureIndex(DocumentNodeStore store) throws
CommitFailedException {
+ NodeBuilder builder = store.getRoot().builder();
+ NodeBuilder idxb = childBuilder(builder, TEST_INDEX_PATH);
+
+ IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(idxb);
+ defnb.indexRule("nt:base").property("foo").propertyIndex();
+ defnb.build();
+
+ merge(store, builder);
+ }
+
private DocumentNodeStore getNodeStore(){
return
builderProvider.newBuilder().setMongoDB(getConnection().getDB()).getNodeStore();
}
@@ -193,7 +209,7 @@ public class DocumentStoreIndexerIT exte
return conn;
}
- private static void configureBunding(DocumentNodeStore store) throws
CommitFailedException {
+ private static void configureBundling(DocumentNodeStore store) throws
CommitFailedException {
NodeState registryState = BundledTypesRegistry.builder()
.forType("app:Asset")
.include("jcr:content")
