Author: chetanm
Date: Wed Sep 6 06:00:26 2017
New Revision: 1807439
URL: http://svn.apache.org/viewvc?rev=1807439&view=rev
Log:
OAK-6353 - Use Document order traversal for reindexing performed on
DocumentNodeStore setups
Add bundling support
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java?rev=1807439&r1=1807438&r2=1807439&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexer.java
Wed Sep 6 06:00:26 2017
@@ -97,10 +97,12 @@ public class DocumentStoreIndexer implem
continue;
}
- //TODO Handle bundled NodeStates
-
NodeStateEntry entry = new NodeStateEntry(nodeState, path);
indexer.index(entry);
+
+ for (DocumentNodeState bundledState :
nodeState.getAllBundledNodesStates()) {
+ indexer.index(new NodeStateEntry(bundledState,
bundledState.getPath()));
+ }
}
progressReporter.reindexingTraversalEnd();
@@ -180,7 +182,7 @@ public class DocumentStoreIndexer implem
traversalLog.trace(id);
}
- private CompositeIndexer prepareIndexers() {
+ protected CompositeIndexer prepareIndexers() {
NodeState root = indexHelper.getNodeStore().getRoot();
List<NodeStateIndexer> indexers = new ArrayList<>();
for (String indexPath : indexHelper.getIndexPaths()) {
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java?rev=1807439&r1=1807438&r2=1807439&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/DocumentStoreIndexerIT.java
Wed Sep 6 06:00:26 2017
@@ -20,22 +20,56 @@
package org.apache.jackrabbit.oak.index;
import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.index.indexer.document.CompositeIndexer;
+import org.apache.jackrabbit.oak.index.indexer.document.DocumentStoreIndexer;
+import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
+import org.apache.jackrabbit.oak.index.indexer.document.NodeStateIndexer;
+import org.apache.jackrabbit.oak.plugins.document.Collection;
+import org.apache.jackrabbit.oak.plugins.document.DocumentMK;
import org.apache.jackrabbit.oak.plugins.document.DocumentMKBuilderProvider;
+import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
+import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
import org.apache.jackrabbit.oak.plugins.document.MongoConnectionFactory;
import org.apache.jackrabbit.oak.plugins.document.MongoUtils;
+import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
+import org.apache.jackrabbit.oak.plugins.document.bundlor.BundledTypesRegistry;
+import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexRootDirectory;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.LocalIndexDir;
-import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.whiteboard.DefaultWhiteboard;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyMap;
+import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE;
+import static org.apache.jackrabbit.oak.plugins.document.TestUtils.createChild;
+import static
org.apache.jackrabbit.oak.plugins.document.bundlor.BundlingConfigHandler.BUNDLOR;
+import static
org.apache.jackrabbit.oak.plugins.document.bundlor.BundlingConfigHandler.DOCUMENT_NODE_STORE;
+import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeNotNull;
@@ -53,10 +87,12 @@ public class DocumentStoreIndexerIT exte
@Test
public void indexMongoRepo() throws Exception{
- fixture = new RepositoryFixture(temporaryFolder.getRoot(),
getNodeStore());
+ DocumentNodeStore dns = getNodeStore();
+ fixture = new RepositoryFixture(temporaryFolder.getRoot(), dns);
createTestData(false);
String checkpoint =
fixture.getNodeStore().checkpoint(TimeUnit.HOURS.toMillis(24));
fixture.close();
+ dns.dispose();
IndexCommand command = new IndexCommand();
@@ -83,15 +119,136 @@ public class DocumentStoreIndexerIT exte
assertEquals(1, idxDirs.size());
}
- private NodeStore getNodeStore(){
+ @Test
+ public void bundling() throws Exception{
+ DocumentMK.Builder docBuilder =
builderProvider.newBuilder().setMongoDB(getConnection().getDB());
+ DocumentNodeStore store = docBuilder.getNodeStore();
+
+ Whiteboard wb = new DefaultWhiteboard();
+ MongoDocumentStore ds = (MongoDocumentStore)
docBuilder.getDocumentStore();
+ wb.register(MongoDocumentStore.class, ds, emptyMap());
+ wb.register(StatisticsProvider.class, StatisticsProvider.NOOP,
emptyMap());
+
+ configureBunding(store);
+
+ NodeBuilder builder = store.getRoot().builder();
+ NodeBuilder appNB = newNode("app:Asset");
+ createChild(appNB,
+ "jcr:content",
+ "jcr:content/comments", //not bundled
+ "jcr:content/metadata",
+ "jcr:content/metadata/xmp", //not bundled
+ "jcr:content/renditions", //includes all
+ "jcr:content/renditions/original",
+ "jcr:content/renditions/original/jcr:content"
+ );
+ builder.child("test").setChildNode("book.jpg", appNB.getNodeState());
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ //Check that bundling is working
+ assertNull(getNodeDocument(ds, "/test/book.jpg/jcr:content"));
+ assertNotNull(getNodeDocument(ds, "/test/book.jpg"));
+
+ String checkpoint = store.checkpoint(100000);
+
+ IndexHelper helper = new IndexHelper(store, store.getBlobStore(), wb,
temporaryFolder.newFolder(),
+ temporaryFolder.newFolder(), asList("/oak:index/foo"));
+ IndexerSupport support = new IndexerSupport(helper, checkpoint);
+
+ CollectingIndexer testIndexer = new CollectingIndexer(p ->
p.startsWith("/test"));
+ DocumentStoreIndexer index = new DocumentStoreIndexer(helper, support)
{
+ @Override
+ protected CompositeIndexer prepareIndexers() {
+ return new CompositeIndexer(asList(testIndexer));
+ }
+ };
+
+
+ index.reindex();
+
+ assertThat(testIndexer.paths, containsInAnyOrder(
+ "/test",
+ "/test/book.jpg",
+ "/test/book.jpg/jcr:content",
+ "/test/book.jpg/jcr:content/comments",
+ "/test/book.jpg/jcr:content/metadata",
+ "/test/book.jpg/jcr:content/metadata/xmp",
+ "/test/book.jpg/jcr:content/renditions",
+ "/test/book.jpg/jcr:content/renditions/original",
+ "/test/book.jpg/jcr:content/renditions/original/jcr:content"
+ ));
+
+ store.dispose();
+
+ }
+
+ private DocumentNodeStore getNodeStore(){
return
builderProvider.newBuilder().setMongoDB(getConnection().getDB()).getNodeStore();
}
private MongoConnection getConnection(){
MongoConnection conn = connectionFactory.getConnection();
assumeNotNull(conn);
- conn.getDB().dropDatabase();
+ MongoUtils.dropCollections(conn.getDB());
return conn;
}
+ private static void configureBunding(DocumentNodeStore store) throws
CommitFailedException {
+ NodeState registryState = BundledTypesRegistry.builder()
+ .forType("app:Asset")
+ .include("jcr:content")
+ .include("jcr:content/metadata")
+ .include("jcr:content/renditions")
+ .include("jcr:content/renditions/**")
+ .build();
+ NodeBuilder builder = store.getRoot().builder();
+ new InitialContent().initialize(builder);
+ builder.getChildNode("jcr:system")
+ .getChildNode(DOCUMENT_NODE_STORE)
+ .getChildNode(BUNDLOR)
+ .setChildNode("app:Asset",
registryState.getChildNode("app:Asset"));
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+
+ private NodeDocument getNodeDocument(DocumentStore ds, String path) {
+ return ds.find(Collection.NODES, Utils.getIdFromPath(path));
+ }
+
+ private static NodeBuilder newNode(String typeName){
+ NodeBuilder builder = EMPTY_NODE.builder();
+ builder.setProperty(JCR_PRIMARYTYPE, typeName);
+ return builder;
+ }
+
+ private static class CollectingIndexer implements NodeStateIndexer {
+ private final Predicate<String> p;
+ List<String> paths = new ArrayList<>();
+
+ private CollectingIndexer(Predicate<String> p) {
+ this.p = p;
+ }
+
+ @Override
+ public boolean shouldInclude(String path) {
+ return true;
+ }
+
+ @Override
+ public boolean shouldInclude(NodeDocument doc) {
+ return true;
+ }
+
+ @Override
+ public void index(NodeStateEntry entry) throws IOException,
CommitFailedException {
+ if (p.test(entry.getPath())) {
+ paths.add(entry.getPath());
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+ }
+
}
