Author: chetanm
Date: Fri May 19 08:11:02 2017
New Revision: 1795593

URL: http://svn.apache.org/viewvc?rev=1795593&view=rev
Log:
OAK-6117 - Enable lucene indexing via oak-run

Initial implementation which connects to the repo in read-write mode
and just triggers the async indexer

Added:
    
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
   (with props)
    
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java
   (with props)
    
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
   (with props)
    
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
   (with props)
Modified:
    
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
    
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
    
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexHelper.java
    
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
    
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
    
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/RepositoryFixture.java

Modified: 
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
 Fri May 19 08:11:02 2017
@@ -1445,7 +1445,7 @@ public class AsyncIndexUpdate implements
         this.mbeanRegistration = mbeanRegistration;
     }
 
-    protected String getName() {
+    public String getName() {
         return name;
     }
 

Modified: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
 Fri May 19 08:11:02 2017
@@ -23,13 +23,16 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
 
+import com.google.common.io.Closer;
 import joptsimple.OptionParser;
 import org.apache.felix.inventory.Format;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
 import org.apache.jackrabbit.oak.console.NodeStoreFixture;
 import org.apache.jackrabbit.oak.run.cli.CommonOptions;
 import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
 import org.apache.jackrabbit.oak.run.cli.Options;
 import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
 import org.apache.jackrabbit.oak.spi.state.NodeStore;
 
 public class IndexCommand implements Command {
@@ -57,8 +60,11 @@ public class IndexCommand implements Com
 
         IndexOptions indexOpts = opts.getOptionBean(IndexOptions.class);
 
-        try (NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts)) 
{
-            execute(fixture.getStore(), indexOpts);
+        NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts);
+        try (Closer closer = Closer.create()) {
+            closer.register(fixture);
+
+            execute(fixture.getStore(), fixture.getBlobStore(), indexOpts, 
closer);
             tellReportPaths();
         }
     }
@@ -77,14 +83,24 @@ public class IndexCommand implements Com
         }
     }
 
-    private void execute(NodeStore store, IndexOptions indexOpts) throws 
IOException {
-        IndexHelper indexHelper = new IndexHelper(store, indexOpts.getOutDir(),
+    private void execute(NodeStore store, BlobStore blobStore, IndexOptions 
indexOpts, Closer closer) throws IOException, CommitFailedException {
+        IndexHelper indexHelper = new IndexHelper(store, blobStore, 
indexOpts.getOutDir(),
                 indexOpts.getWorkDir(), indexOpts.getIndexPaths());
 
+        closer.register(indexHelper);
+
         dumpIndexStats(indexOpts, indexHelper);
         dumpIndexDefinitions(indexOpts, indexHelper);
         performConsistencyCheck(indexOpts, indexHelper);
         dumpIndexContents(indexOpts, indexHelper);
+        reindexIndex(indexOpts, indexHelper);
+    }
+
+    private void reindexIndex(IndexOptions indexOpts, IndexHelper indexHelper) 
throws IOException, CommitFailedException {
+        if (!indexOpts.isReindex()){
+            return;
+        }
+        new ReIndexer(indexHelper).reindex();
     }
 
     private void dumpIndexContents(IndexOptions indexOpts, IndexHelper 
indexHelper) throws IOException {

Modified: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexHelper.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexHelper.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexHelper.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexHelper.java
 Fri May 19 08:11:02 2017
@@ -19,9 +19,23 @@
 
 package org.apache.jackrabbit.oak.index;
 
+import java.io.Closeable;
 import java.io.File;
+import java.io.IOException;
 import java.util.List;
-
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.Closer;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
 import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
 import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoServiceImpl;
 import org.apache.jackrabbit.oak.plugins.index.IndexInfoService;
@@ -32,9 +46,16 @@ import org.apache.jackrabbit.oak.plugins
 import org.apache.jackrabbit.oak.plugins.index.inventory.IndexPrinter;
 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexInfoProvider;
 import 
org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexInfoProvider;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
+import org.apache.jackrabbit.oak.spi.mount.Mounts;
 import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-class IndexHelper {
+class IndexHelper implements Closeable{
+    private final Logger log = LoggerFactory.getLogger(getClass());
     private final NodeStore store;
     private final File outputDir;
     private final File workDir;
@@ -42,12 +63,17 @@ class IndexHelper {
     private IndexPathService indexPathService;
     private AsyncIndexInfoService asyncIndexInfoService;
     private final List<String> indexPaths;
+    private LuceneIndexHelper luceneIndexHelper;
+    private Executor executor;
+    private final Closer closer = Closer.create();
+    private final BlobStore blobStore;
 
-    IndexHelper(NodeStore store, File outputDir, File workDir, List<String> 
indexPaths) {
+    IndexHelper(NodeStore store, BlobStore blobStore, File outputDir, File 
workDir, List<String> indexPaths) {
         this.store = store;
+        this.blobStore = blobStore;
         this.outputDir = outputDir;
         this.workDir = workDir;
-        this.indexPaths = indexPaths;
+        this.indexPaths = ImmutableList.copyOf(indexPaths);
     }
 
     public NodeStore getNodeStore() {
@@ -81,6 +107,44 @@ class IndexHelper {
         return indexPathService;
     }
 
+    public List<String> getIndexPaths() {
+        return indexPaths;
+    }
+
+    public Executor getExecutor() {
+        if (executor == null) {
+            ExecutorService executorService = createExecutor();
+            closer.register(new ExecutorCloser(executorService));
+            executor = executorService;
+        }
+        return executor;
+    }
+
+    public MountInfoProvider getMountInfoProvider(){
+        return Mounts.defaultMountInfoProvider();
+    }
+
+    public StatisticsProvider getStatisticsProvider(){
+        return StatisticsProvider.NOOP; //TODO Wire in a real stats provider 
based on metric
+    }
+
+    public BlobStore getBlobStore() {
+        return blobStore;
+    }
+
+    public LuceneIndexHelper getLuceneIndexHelper(){
+        if (luceneIndexHelper == null) {
+            luceneIndexHelper = new LuceneIndexHelper(this);
+            closer.register(luceneIndexHelper);
+        }
+        return luceneIndexHelper;
+    }
+
+    @Override
+    public void close() throws IOException {
+        closer.close();
+    }
+
     private AsyncIndexInfoService getAsyncIndexInfoService() {
         if (asyncIndexInfoService == null) {
             asyncIndexInfoService = new AsyncIndexInfoServiceImpl(store);
@@ -100,4 +164,28 @@ class IndexHelper {
         indexInfoService.bindInfoProviders(new LuceneIndexInfoProvider(store, 
getAsyncIndexInfoService(), workDir));
         indexInfoService.bindInfoProviders(new 
PropertyIndexInfoProvider(store));
     }
+
+    private ThreadPoolExecutor createExecutor() {
+        ThreadPoolExecutor executor = new ThreadPoolExecutor(0, 5, 60L, 
TimeUnit.SECONDS,
+                new LinkedBlockingQueue<Runnable>(), new ThreadFactory() {
+            private final AtomicInteger counter = new AtomicInteger();
+            private final Thread.UncaughtExceptionHandler handler =
+                    (t, e) -> log.warn("Error occurred in asynchronous 
processing ", e);
+            @Override
+            public Thread newThread(@Nonnull Runnable r) {
+                Thread thread = new Thread(r, createName());
+                thread.setDaemon(true);
+                thread.setPriority(Thread.MIN_PRIORITY);
+                thread.setUncaughtExceptionHandler(handler);
+                return thread;
+            }
+
+            private String createName() {
+                return "oak-lucene-" + counter.getAndIncrement();
+            }
+        });
+        executor.setKeepAliveTime(1, TimeUnit.MINUTES);
+        executor.allowCoreThreadTimeOut(true);
+        return executor;
+    }
 }

Modified: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
 Fri May 19 08:11:02 2017
@@ -51,6 +51,7 @@ public class IndexOptions implements Opt
     private final OptionSpec<Void> stats;
     private final OptionSpec<Void> definitions;
     private final OptionSpec<Void> dumpIndex;
+    private final OptionSpec<Void> reindex;
     private final OptionSpec<Integer> consistencyCheck;
     private OptionSet options;
     private final Set<OptionSpec> actionOpts;
@@ -75,9 +76,10 @@ public class IndexOptions implements Opt
                 .withOptionalArg().ofType(Integer.class).defaultsTo(1);
 
         dumpIndex = parser.accepts("index-dump", "Dumps index content");
+        reindex = parser.accepts("reindex", "Reindex the 
indexes").availableIf("index-paths");
 
         //Set of options which define action
-        actionOpts = ImmutableSet.of(stats, definitions, consistencyCheck, 
dumpIndex);
+        actionOpts = ImmutableSet.of(stats, definitions, consistencyCheck, 
dumpIndex, reindex);
         operationNames = collectionOperationNames(actionOpts);
     }
 
@@ -139,6 +141,10 @@ public class IndexOptions implements Opt
         return consistencyCheck.value(options);
     }
 
+    public boolean isReindex() {
+        return options.has(reindex);
+    }
+
     public List<String> getIndexPaths(){
         return options.has(indexPaths) ? trim(indexPaths.values(options)) : 
Collections.emptyList();
     }

Added: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java?rev=1795593&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
 Fri May 19 08:11:02 2017
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+
+class LuceneIndexHelper implements Closeable {
+    private final IndexHelper indexHelper;
+    private IndexCopier indexCopier;
+    private final ExtractedTextCache textCache =
+            new ExtractedTextCache(FileUtils.ONE_MB * 5, 
TimeUnit.HOURS.toSeconds(5));
+
+    LuceneIndexHelper(IndexHelper indexHelper) {
+        this.indexHelper = indexHelper;
+    }
+
+    public LuceneIndexEditorProvider createEditorProvider() throws IOException 
{
+        LuceneIndexEditorProvider editor =  new LuceneIndexEditorProvider(
+                getIndexCopier(),
+                textCache,
+                null,
+                indexHelper.getMountInfoProvider()
+        );
+
+        if (indexHelper.getBlobStore() instanceof GarbageCollectableBlobStore) 
{
+            editor.setBlobStore((GarbageCollectableBlobStore) 
indexHelper.getBlobStore());
+        }
+
+        return editor;
+    }
+
+    private IndexCopier getIndexCopier() throws IOException {
+        if (indexCopier == null) {
+            File indexWorkDir = new File(indexHelper.getWorkDir(), 
"indexWorkDir");
+            indexCopier = new IndexCopier(indexHelper.getExecutor(), 
indexWorkDir, true);
+        }
+        return indexCopier;
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (indexCopier != null) {
+            indexCopier.close();
+        }
+    }
+}

Propchange: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java?rev=1795593&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java
 Fri May 19 08:11:02 2017
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index;
+
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.commit.AnnotatingConflictHandler;
+import org.apache.jackrabbit.oak.plugins.commit.ConflictHook;
+import org.apache.jackrabbit.oak.plugins.commit.ConflictValidatorProvider;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider;
+import org.apache.jackrabbit.oak.spi.commit.CompositeHook;
+import org.apache.jackrabbit.oak.spi.commit.EditorHook;
+import org.apache.jackrabbit.oak.spi.commit.EditorProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+class NodeStoreUtils {
+
+    public static NodeBuilder childBuilder(NodeBuilder nb, String path) {
+        for (String name : PathUtils.elements(checkNotNull(path))) {
+            nb = nb.child(name);
+        }
+        return nb;
+    }
+
+    public static void mergeWithConcurrentCheck(NodeStore nodeStore, 
NodeBuilder builder) throws CommitFailedException {
+        List<EditorProvider> editorProviders = Lists.newArrayList();
+        editorProviders.add(new ConflictValidatorProvider());
+        CompositeHook hooks = new CompositeHook(
+                new ConflictHook(new AnnotatingConflictHandler()),
+                new 
EditorHook(CompositeEditorProvider.compose(editorProviders)));
+
+        nodeStore.merge(builder, hooks, CommitInfo.EMPTY);
+    }
+
+}

Propchange: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java?rev=1795593&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
 Fri May 19 08:11:02 2017
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index;
+
+import java.io.IOException;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
+import 
org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider;
+import 
org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Arrays.asList;
+
+class ReIndexer {
+    private final IndexHelper indexHelper;
+
+    public ReIndexer(IndexHelper indexHelper) {
+        this.indexHelper = checkNotNull(indexHelper);
+    }
+
+    public void reindex() throws IOException, CommitFailedException {
+        new SimpleAsyncReindexer(indexHelper, indexHelper.getIndexPaths(), 
createIndexEditorProvider()).reindex();
+    }
+
+    private IndexEditorProvider createIndexEditorProvider() throws IOException 
{
+        //Need to list all used editors otherwise async index run fails with
+        //MissingIndexEditor exception. Better approach would be to change 
lane for
+        //those indexes and then do reindexing
+        NodeCounterEditorProvider counter = new NodeCounterEditorProvider();
+        IndexEditorProvider lucene = 
indexHelper.getLuceneIndexHelper().createEditorProvider();
+        IndexEditorProvider property = new 
PropertyIndexEditorProvider().with(indexHelper.getMountInfoProvider());
+
+        return CompositeIndexEditorProvider.compose(asList(lucene, property, 
counter));
+    }
+}

Propchange: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java?rev=1795593&view=auto
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
 (added)
+++ 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
 Fri May 19 08:11:02 2017
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index;
+
+import java.util.List;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.jackrabbit.oak.index.NodeStoreUtils.childBuilder;
+import static 
org.apache.jackrabbit.oak.index.NodeStoreUtils.mergeWithConcurrentCheck;
+
+public class SimpleAsyncReindexer {
+    private final Logger log = LoggerFactory.getLogger(getClass());
+    private final IndexHelper indexHelper;
+    private final List<String> indexPaths;
+    private final IndexEditorProvider indexEditorProvider;
+
+    public SimpleAsyncReindexer(IndexHelper indexHelper, List<String> 
indexPaths, IndexEditorProvider indexEditorProvider) {
+        this.indexHelper = indexHelper;
+        this.indexPaths = indexPaths;
+        this.indexEditorProvider = indexEditorProvider;
+    }
+
+    public void reindex() throws CommitFailedException {
+        setReindexFlag();
+        Multimap<String, String> laneMapping = getIndexesPerLane();
+        for (String laneName : laneMapping.keySet()) {
+            reindex(laneName);
+        }
+    }
+
+    private void setReindexFlag() throws CommitFailedException {
+        NodeBuilder builder = getNodeStore().getRoot().builder();
+        for (String indexPath : indexPaths) {
+            NodeBuilder idx = childBuilder(builder, indexPath);
+            idx.setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true);
+        }
+        mergeWithConcurrentCheck(getNodeStore(), builder);
+    }
+
+    private void reindex(String laneName) {
+        AsyncIndexUpdate async = new AsyncIndexUpdate(
+                laneName,
+                getNodeStore(),
+                indexEditorProvider,
+                indexHelper.getStatisticsProvider(),
+                false);
+
+        //TODO Expose the JMX
+        boolean done = false;
+        while (!done) {
+            //TODO Check for timeout
+            async.run();
+            done = async.isFinished();
+        }
+    }
+
+    private Multimap<String, String> getIndexesPerLane(){
+        NodeState root = getNodeStore().getRoot();
+        Multimap<String, String> map = HashMultimap.create();
+        for (String indexPath : indexPaths) {
+            NodeState idxState = NodeStateUtils.getNode(root, indexPath);
+            String asyncName = IndexUtils.getAsyncLaneName(idxState, 
indexPath);
+            if (asyncName != null) {
+                map.put(asyncName, indexPath);
+            } else {
+                log.warn("No async value for indexPath {}. Ignoring it", 
indexPath);
+            }
+        }
+        return map;
+    }
+
+    private NodeStore getNodeStore() {
+        return indexHelper.getNodeStore();
+    }
+}

Propchange: 
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: 
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
 Fri May 19 08:11:02 2017
@@ -28,8 +28,11 @@ import javax.jcr.RepositoryException;
 import javax.jcr.Session;
 
 import com.google.common.io.Files;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
 import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
 import 
org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
 import org.junit.After;
 import org.junit.Rule;
 import org.junit.Test;
@@ -37,8 +40,10 @@ import org.junit.rules.TemporaryFolder;
 
 import static java.nio.charset.Charset.defaultCharset;
 import static org.apache.jackrabbit.commons.JcrUtils.getOrCreateByPath;
+import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
@@ -58,7 +63,7 @@ public class IndexCommandIT {
 
     @Test
     public void dumpStatsAndInfo() throws Exception{
-        createTestData();
+        createTestData(false);
         //Close the repository so as all changes are flushed
         fixture.close();
 
@@ -87,7 +92,7 @@ public class IndexCommandIT {
 
     @Test
     public void selectedIndexPaths() throws Exception{
-        createTestData();
+        createTestData(false);
         //Close the repository so as all changes are flushed
         fixture.close();
 
@@ -115,7 +120,7 @@ public class IndexCommandIT {
 
     @Test
     public void consistencyCheck() throws Exception{
-        createTestData();
+        createTestData(false);
         //Close the repository so as all changes are flushed
         fixture.close();
 
@@ -143,7 +148,7 @@ public class IndexCommandIT {
 
     @Test
     public void dumpIndex() throws Exception{
-        createTestData();
+        createTestData(false);
         //Close the repository so as all changes are flushed
         fixture.close();
 
@@ -163,10 +168,39 @@ public class IndexCommandIT {
         assertTrue(dumpDir.exists());
     }
 
-    private void createTestData() throws IOException, RepositoryException {
+    @Test
+    public void reindex() throws Exception{
+        createTestData(true);
+        fixture.getAsyncIndexUpdate("async").run();
+        //Close the repository so as all changes are flushed
+        fixture.close();
+
+        IndexCommand command = new IndexCommand();
+
+        File outDir = temporaryFolder.newFolder();
+        File storeDir = fixture.getDir();
+        String[] args = {
+                "--index-work-dir=" + 
temporaryFolder.newFolder().getAbsolutePath(),
+                "--index-out-dir="  + outDir.getAbsolutePath(),
+                "--index-paths=/oak:index/fooIndex",
+                "--read-write=true",
+                "--reindex",
+                "--", // -- indicates that options have ended and rest needs 
to be treated as non option
+                storeDir.getAbsolutePath()
+        };
+
+        command.execute(args);
+
+        RepositoryFixture fixture2 = new RepositoryFixture(storeDir);
+        NodeStore store2 = fixture2.getNodeStore();
+        PropertyState reindexCount = getNode(store2.getRoot(), 
"/oak:index/fooIndex").getProperty(IndexConstants.REINDEX_COUNT);
+        assertEquals(2, reindexCount.getValue(Type.LONG).longValue());
+    }
+
+    private void createTestData(boolean asyncIndex) throws IOException, 
RepositoryException {
         fixture = new RepositoryFixture(temporaryFolder.newFolder());
         indexIndexDefinitions();
-        createLuceneIndex();
+        createLuceneIndex(asyncIndex);
         addTestContent();
     }
 
@@ -190,9 +224,11 @@ public class IndexCommandIT {
         session.logout();
     }
 
-    private void createLuceneIndex() throws IOException, RepositoryException {
+    private void createLuceneIndex(boolean asyncIndex) throws IOException, 
RepositoryException {
         IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder();
-        idxBuilder.noAsync();
+        if (!asyncIndex) {
+            idxBuilder.noAsync();
+        }
         idxBuilder.indexRule("nt:base").property("foo").propertyIndex();
 
         Session session = fixture.getAdminSession();

Modified: 
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/RepositoryFixture.java
URL: 
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/RepositoryFixture.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
--- 
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/RepositoryFixture.java
 (original)
+++ 
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/RepositoryFixture.java
 Fri May 19 08:11:02 2017
@@ -31,6 +31,7 @@ import javax.jcr.SimpleCredentials;
 import org.apache.jackrabbit.api.JackrabbitRepository;
 import org.apache.jackrabbit.oak.Oak;
 import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
 import 
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
 import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
 import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
@@ -40,12 +41,17 @@ import org.apache.jackrabbit.oak.segment
 import org.apache.jackrabbit.oak.spi.commit.Observer;
 import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
 import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils;
+
+import static 
org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.getService;
 
 public class RepositoryFixture implements Closeable {
     private final File dir;
     private Repository repository;
     private FileStore fileStore;
     private NodeStore nodeStore;
+    private Whiteboard whiteboard;
 
     public RepositoryFixture(File dir) {
         this.dir = dir;
@@ -69,6 +75,11 @@ public class RepositoryFixture implement
         return nodeStore;
     }
 
+    public AsyncIndexUpdate getAsyncIndexUpdate(String laneName) {
+        return (AsyncIndexUpdate) getService(whiteboard, Runnable.class,
+                (runnable) -> runnable instanceof AsyncIndexUpdate && 
laneName.equals(((AsyncIndexUpdate)runnable).getName()));
+    }
+
     @Override
     public void close() throws IOException {
         if (repository instanceof JackrabbitRepository) {
@@ -80,6 +91,8 @@ public class RepositoryFixture implement
             fileStore.close();
             fileStore = null;
         }
+
+        whiteboard = null;
     }
 
     public File getDir() {
@@ -88,10 +101,14 @@ public class RepositoryFixture implement
 
     private Repository createRepository() throws IOException {
         Oak oak = new Oak(getNodeStore());
+
+        oak.withAsyncIndexing("async", 3600); //Effectively disable async 
indexing
         configureLuceneProvider(oak);
 
         Jcr jcr = new Jcr(oak);
-        return jcr.createRepository();
+        Repository repository = jcr.createRepository();
+        whiteboard = oak.getWhiteboard();
+        return repository;
     }
 
     private void configureLuceneProvider(Oak oak) throws IOException {


Reply via email to