Author: chetanm
Date: Fri May 19 08:11:02 2017
New Revision: 1795593
URL: http://svn.apache.org/viewvc?rev=1795593&view=rev
Log:
OAK-6117 - Enable lucene indexing via oak-run
Initial implementation which connects to the repo in read-write mode
and just triggers the async indexer
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
(with props)
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java
(with props)
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
(with props)
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexHelper.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/RepositoryFixture.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
Fri May 19 08:11:02 2017
@@ -1445,7 +1445,7 @@ public class AsyncIndexUpdate implements
this.mbeanRegistration = mbeanRegistration;
}
- protected String getName() {
+ public String getName() {
return name;
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
Fri May 19 08:11:02 2017
@@ -23,13 +23,16 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
+import com.google.common.io.Closer;
import joptsimple.OptionParser;
import org.apache.felix.inventory.Format;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.console.NodeStoreFixture;
import org.apache.jackrabbit.oak.run.cli.CommonOptions;
import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
import org.apache.jackrabbit.oak.run.cli.Options;
import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
public class IndexCommand implements Command {
@@ -57,8 +60,11 @@ public class IndexCommand implements Com
IndexOptions indexOpts = opts.getOptionBean(IndexOptions.class);
- try (NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts))
{
- execute(fixture.getStore(), indexOpts);
+ NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts);
+ try (Closer closer = Closer.create()) {
+ closer.register(fixture);
+
+ execute(fixture.getStore(), fixture.getBlobStore(), indexOpts,
closer);
tellReportPaths();
}
}
@@ -77,14 +83,24 @@ public class IndexCommand implements Com
}
}
- private void execute(NodeStore store, IndexOptions indexOpts) throws
IOException {
- IndexHelper indexHelper = new IndexHelper(store, indexOpts.getOutDir(),
+ private void execute(NodeStore store, BlobStore blobStore, IndexOptions
indexOpts, Closer closer) throws IOException, CommitFailedException {
+ IndexHelper indexHelper = new IndexHelper(store, blobStore,
indexOpts.getOutDir(),
indexOpts.getWorkDir(), indexOpts.getIndexPaths());
+ closer.register(indexHelper);
+
dumpIndexStats(indexOpts, indexHelper);
dumpIndexDefinitions(indexOpts, indexHelper);
performConsistencyCheck(indexOpts, indexHelper);
dumpIndexContents(indexOpts, indexHelper);
+ reindexIndex(indexOpts, indexHelper);
+ }
+
+ private void reindexIndex(IndexOptions indexOpts, IndexHelper indexHelper)
throws IOException, CommitFailedException {
+ if (!indexOpts.isReindex()){
+ return;
+ }
+ new ReIndexer(indexHelper).reindex();
}
private void dumpIndexContents(IndexOptions indexOpts, IndexHelper
indexHelper) throws IOException {
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexHelper.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexHelper.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexHelper.java
Fri May 19 08:11:02 2017
@@ -19,9 +19,23 @@
package org.apache.jackrabbit.oak.index;
+import java.io.Closeable;
import java.io.File;
+import java.io.IOException;
import java.util.List;
-
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.Closer;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoServiceImpl;
import org.apache.jackrabbit.oak.plugins.index.IndexInfoService;
@@ -32,9 +46,16 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.inventory.IndexPrinter;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexInfoProvider;
import
org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexInfoProvider;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
+import org.apache.jackrabbit.oak.spi.mount.Mounts;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-class IndexHelper {
+class IndexHelper implements Closeable{
+ private final Logger log = LoggerFactory.getLogger(getClass());
private final NodeStore store;
private final File outputDir;
private final File workDir;
@@ -42,12 +63,17 @@ class IndexHelper {
private IndexPathService indexPathService;
private AsyncIndexInfoService asyncIndexInfoService;
private final List<String> indexPaths;
+ private LuceneIndexHelper luceneIndexHelper;
+ private Executor executor;
+ private final Closer closer = Closer.create();
+ private final BlobStore blobStore;
- IndexHelper(NodeStore store, File outputDir, File workDir, List<String>
indexPaths) {
+ IndexHelper(NodeStore store, BlobStore blobStore, File outputDir, File
workDir, List<String> indexPaths) {
this.store = store;
+ this.blobStore = blobStore;
this.outputDir = outputDir;
this.workDir = workDir;
- this.indexPaths = indexPaths;
+ this.indexPaths = ImmutableList.copyOf(indexPaths);
}
public NodeStore getNodeStore() {
@@ -81,6 +107,44 @@ class IndexHelper {
return indexPathService;
}
+ public List<String> getIndexPaths() {
+ return indexPaths;
+ }
+
+ public Executor getExecutor() {
+ if (executor == null) {
+ ExecutorService executorService = createExecutor();
+ closer.register(new ExecutorCloser(executorService));
+ executor = executorService;
+ }
+ return executor;
+ }
+
+ public MountInfoProvider getMountInfoProvider(){
+ return Mounts.defaultMountInfoProvider();
+ }
+
+ public StatisticsProvider getStatisticsProvider(){
+ return StatisticsProvider.NOOP; //TODO Wire in a real stats provider
based on metric
+ }
+
+ public BlobStore getBlobStore() {
+ return blobStore;
+ }
+
+ public LuceneIndexHelper getLuceneIndexHelper(){
+ if (luceneIndexHelper == null) {
+ luceneIndexHelper = new LuceneIndexHelper(this);
+ closer.register(luceneIndexHelper);
+ }
+ return luceneIndexHelper;
+ }
+
+ @Override
+ public void close() throws IOException {
+ closer.close();
+ }
+
private AsyncIndexInfoService getAsyncIndexInfoService() {
if (asyncIndexInfoService == null) {
asyncIndexInfoService = new AsyncIndexInfoServiceImpl(store);
@@ -100,4 +164,28 @@ class IndexHelper {
indexInfoService.bindInfoProviders(new LuceneIndexInfoProvider(store,
getAsyncIndexInfoService(), workDir));
indexInfoService.bindInfoProviders(new
PropertyIndexInfoProvider(store));
}
+
+ private ThreadPoolExecutor createExecutor() {
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(0, 5, 60L,
TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>(), new ThreadFactory() {
+ private final AtomicInteger counter = new AtomicInteger();
+ private final Thread.UncaughtExceptionHandler handler =
+ (t, e) -> log.warn("Error occurred in asynchronous
processing ", e);
+ @Override
+ public Thread newThread(@Nonnull Runnable r) {
+ Thread thread = new Thread(r, createName());
+ thread.setDaemon(true);
+ thread.setPriority(Thread.MIN_PRIORITY);
+ thread.setUncaughtExceptionHandler(handler);
+ return thread;
+ }
+
+ private String createName() {
+ return "oak-lucene-" + counter.getAndIncrement();
+ }
+ });
+ executor.setKeepAliveTime(1, TimeUnit.MINUTES);
+ executor.allowCoreThreadTimeOut(true);
+ return executor;
+ }
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
Fri May 19 08:11:02 2017
@@ -51,6 +51,7 @@ public class IndexOptions implements Opt
private final OptionSpec<Void> stats;
private final OptionSpec<Void> definitions;
private final OptionSpec<Void> dumpIndex;
+ private final OptionSpec<Void> reindex;
private final OptionSpec<Integer> consistencyCheck;
private OptionSet options;
private final Set<OptionSpec> actionOpts;
@@ -75,9 +76,10 @@ public class IndexOptions implements Opt
.withOptionalArg().ofType(Integer.class).defaultsTo(1);
dumpIndex = parser.accepts("index-dump", "Dumps index content");
+ reindex = parser.accepts("reindex", "Reindex the
indexes").availableIf("index-paths");
//Set of options which define action
- actionOpts = ImmutableSet.of(stats, definitions, consistencyCheck,
dumpIndex);
+ actionOpts = ImmutableSet.of(stats, definitions, consistencyCheck,
dumpIndex, reindex);
operationNames = collectionOperationNames(actionOpts);
}
@@ -139,6 +141,10 @@ public class IndexOptions implements Opt
return consistencyCheck.value(options);
}
+ public boolean isReindex() {
+ return options.has(reindex);
+ }
+
public List<String> getIndexPaths(){
return options.has(indexPaths) ? trim(indexPaths.values(options)) :
Collections.emptyList();
}
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java?rev=1795593&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
(added)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
Fri May 19 08:11:02 2017
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+
+class LuceneIndexHelper implements Closeable {
+ private final IndexHelper indexHelper;
+ private IndexCopier indexCopier;
+ private final ExtractedTextCache textCache =
+ new ExtractedTextCache(FileUtils.ONE_MB * 5,
TimeUnit.HOURS.toSeconds(5));
+
+ LuceneIndexHelper(IndexHelper indexHelper) {
+ this.indexHelper = indexHelper;
+ }
+
+ public LuceneIndexEditorProvider createEditorProvider() throws IOException
{
+ LuceneIndexEditorProvider editor = new LuceneIndexEditorProvider(
+ getIndexCopier(),
+ textCache,
+ null,
+ indexHelper.getMountInfoProvider()
+ );
+
+ if (indexHelper.getBlobStore() instanceof GarbageCollectableBlobStore)
{
+ editor.setBlobStore((GarbageCollectableBlobStore)
indexHelper.getBlobStore());
+ }
+
+ return editor;
+ }
+
+ private IndexCopier getIndexCopier() throws IOException {
+ if (indexCopier == null) {
+ File indexWorkDir = new File(indexHelper.getWorkDir(),
"indexWorkDir");
+ indexCopier = new IndexCopier(indexHelper.getExecutor(),
indexWorkDir, true);
+ }
+ return indexCopier;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (indexCopier != null) {
+ indexCopier.close();
+ }
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java?rev=1795593&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java
(added)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java
Fri May 19 08:11:02 2017
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index;
+
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.plugins.commit.AnnotatingConflictHandler;
+import org.apache.jackrabbit.oak.plugins.commit.ConflictHook;
+import org.apache.jackrabbit.oak.plugins.commit.ConflictValidatorProvider;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider;
+import org.apache.jackrabbit.oak.spi.commit.CompositeHook;
+import org.apache.jackrabbit.oak.spi.commit.EditorHook;
+import org.apache.jackrabbit.oak.spi.commit.EditorProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+class NodeStoreUtils {
+
+ public static NodeBuilder childBuilder(NodeBuilder nb, String path) {
+ for (String name : PathUtils.elements(checkNotNull(path))) {
+ nb = nb.child(name);
+ }
+ return nb;
+ }
+
+ public static void mergeWithConcurrentCheck(NodeStore nodeStore,
NodeBuilder builder) throws CommitFailedException {
+ List<EditorProvider> editorProviders = Lists.newArrayList();
+ editorProviders.add(new ConflictValidatorProvider());
+ CompositeHook hooks = new CompositeHook(
+ new ConflictHook(new AnnotatingConflictHandler()),
+ new
EditorHook(CompositeEditorProvider.compose(editorProviders)));
+
+ nodeStore.merge(builder, hooks, CommitInfo.EMPTY);
+ }
+
+}
Propchange:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/NodeStoreUtils.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java?rev=1795593&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
(added)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
Fri May 19 08:11:02 2017
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index;
+
+import java.io.IOException;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
+import
org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider;
+import
org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Arrays.asList;
+
+class ReIndexer {
+ private final IndexHelper indexHelper;
+
+ public ReIndexer(IndexHelper indexHelper) {
+ this.indexHelper = checkNotNull(indexHelper);
+ }
+
+ public void reindex() throws IOException, CommitFailedException {
+ new SimpleAsyncReindexer(indexHelper, indexHelper.getIndexPaths(),
createIndexEditorProvider()).reindex();
+ }
+
+ private IndexEditorProvider createIndexEditorProvider() throws IOException
{
+ //Need to list all used editors otherwise async index run fails with
+ //MissingIndexEditor exception. Better approach would be to change
lane for
+ //those indexes and then do reindexing
+ NodeCounterEditorProvider counter = new NodeCounterEditorProvider();
+ IndexEditorProvider lucene =
indexHelper.getLuceneIndexHelper().createEditorProvider();
+ IndexEditorProvider property = new
PropertyIndexEditorProvider().with(indexHelper.getMountInfoProvider());
+
+ return CompositeIndexEditorProvider.compose(asList(lucene, property,
counter));
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
------------------------------------------------------------------------------
svn:eol-style = native
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java?rev=1795593&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
(added)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
Fri May 19 08:11:02 2017
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index;
+
+import java.util.List;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.jackrabbit.oak.index.NodeStoreUtils.childBuilder;
+import static
org.apache.jackrabbit.oak.index.NodeStoreUtils.mergeWithConcurrentCheck;
+
+public class SimpleAsyncReindexer {
+ private final Logger log = LoggerFactory.getLogger(getClass());
+ private final IndexHelper indexHelper;
+ private final List<String> indexPaths;
+ private final IndexEditorProvider indexEditorProvider;
+
+ public SimpleAsyncReindexer(IndexHelper indexHelper, List<String>
indexPaths, IndexEditorProvider indexEditorProvider) {
+ this.indexHelper = indexHelper;
+ this.indexPaths = indexPaths;
+ this.indexEditorProvider = indexEditorProvider;
+ }
+
+ public void reindex() throws CommitFailedException {
+ setReindexFlag();
+ Multimap<String, String> laneMapping = getIndexesPerLane();
+ for (String laneName : laneMapping.keySet()) {
+ reindex(laneName);
+ }
+ }
+
+ private void setReindexFlag() throws CommitFailedException {
+ NodeBuilder builder = getNodeStore().getRoot().builder();
+ for (String indexPath : indexPaths) {
+ NodeBuilder idx = childBuilder(builder, indexPath);
+ idx.setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true);
+ }
+ mergeWithConcurrentCheck(getNodeStore(), builder);
+ }
+
+ private void reindex(String laneName) {
+ AsyncIndexUpdate async = new AsyncIndexUpdate(
+ laneName,
+ getNodeStore(),
+ indexEditorProvider,
+ indexHelper.getStatisticsProvider(),
+ false);
+
+ //TODO Expose the JMX
+ boolean done = false;
+ while (!done) {
+ //TODO Check for timeout
+ async.run();
+ done = async.isFinished();
+ }
+ }
+
+ private Multimap<String, String> getIndexesPerLane(){
+ NodeState root = getNodeStore().getRoot();
+ Multimap<String, String> map = HashMultimap.create();
+ for (String indexPath : indexPaths) {
+ NodeState idxState = NodeStateUtils.getNode(root, indexPath);
+ String asyncName = IndexUtils.getAsyncLaneName(idxState,
indexPath);
+ if (asyncName != null) {
+ map.put(asyncName, indexPath);
+ } else {
+ log.warn("No async value for indexPath {}. Ignoring it",
indexPath);
+ }
+ }
+ return map;
+ }
+
+ private NodeStore getNodeStore() {
+ return indexHelper.getNodeStore();
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
Fri May 19 08:11:02 2017
@@ -28,8 +28,11 @@ import javax.jcr.RepositoryException;
import javax.jcr.Session;
import com.google.common.io.Files;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
import
org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.After;
import org.junit.Rule;
import org.junit.Test;
@@ -37,8 +40,10 @@ import org.junit.rules.TemporaryFolder;
import static java.nio.charset.Charset.defaultCharset;
import static org.apache.jackrabbit.commons.JcrUtils.getOrCreateByPath;
+import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -58,7 +63,7 @@ public class IndexCommandIT {
@Test
public void dumpStatsAndInfo() throws Exception{
- createTestData();
+ createTestData(false);
//Close the repository so as all changes are flushed
fixture.close();
@@ -87,7 +92,7 @@ public class IndexCommandIT {
@Test
public void selectedIndexPaths() throws Exception{
- createTestData();
+ createTestData(false);
//Close the repository so as all changes are flushed
fixture.close();
@@ -115,7 +120,7 @@ public class IndexCommandIT {
@Test
public void consistencyCheck() throws Exception{
- createTestData();
+ createTestData(false);
//Close the repository so as all changes are flushed
fixture.close();
@@ -143,7 +148,7 @@ public class IndexCommandIT {
@Test
public void dumpIndex() throws Exception{
- createTestData();
+ createTestData(false);
//Close the repository so as all changes are flushed
fixture.close();
@@ -163,10 +168,39 @@ public class IndexCommandIT {
assertTrue(dumpDir.exists());
}
- private void createTestData() throws IOException, RepositoryException {
+ @Test
+ public void reindex() throws Exception{
+ createTestData(true);
+ fixture.getAsyncIndexUpdate("async").run();
+ //Close the repository so as all changes are flushed
+ fixture.close();
+
+ IndexCommand command = new IndexCommand();
+
+ File outDir = temporaryFolder.newFolder();
+ File storeDir = fixture.getDir();
+ String[] args = {
+ "--index-work-dir=" +
temporaryFolder.newFolder().getAbsolutePath(),
+ "--index-out-dir=" + outDir.getAbsolutePath(),
+ "--index-paths=/oak:index/fooIndex",
+ "--read-write=true",
+ "--reindex",
+ "--", // -- indicates that options have ended and rest needs
to be treated as non option
+ storeDir.getAbsolutePath()
+ };
+
+ command.execute(args);
+
+ RepositoryFixture fixture2 = new RepositoryFixture(storeDir);
+ NodeStore store2 = fixture2.getNodeStore();
+ PropertyState reindexCount = getNode(store2.getRoot(),
"/oak:index/fooIndex").getProperty(IndexConstants.REINDEX_COUNT);
+ assertEquals(2, reindexCount.getValue(Type.LONG).longValue());
+ }
+
+ private void createTestData(boolean asyncIndex) throws IOException,
RepositoryException {
fixture = new RepositoryFixture(temporaryFolder.newFolder());
indexIndexDefinitions();
- createLuceneIndex();
+ createLuceneIndex(asyncIndex);
addTestContent();
}
@@ -190,9 +224,11 @@ public class IndexCommandIT {
session.logout();
}
- private void createLuceneIndex() throws IOException, RepositoryException {
+ private void createLuceneIndex(boolean asyncIndex) throws IOException,
RepositoryException {
IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder();
- idxBuilder.noAsync();
+ if (!asyncIndex) {
+ idxBuilder.noAsync();
+ }
idxBuilder.indexRule("nt:base").property("foo").propertyIndex();
Session session = fixture.getAdminSession();
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/RepositoryFixture.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/RepositoryFixture.java?rev=1795593&r1=1795592&r2=1795593&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/RepositoryFixture.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/RepositoryFixture.java
Fri May 19 08:11:02 2017
@@ -31,6 +31,7 @@ import javax.jcr.SimpleCredentials;
import org.apache.jackrabbit.api.JackrabbitRepository;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
import
org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
@@ -40,12 +41,17 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.spi.commit.Observer;
import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils;
+
+import static
org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.getService;
public class RepositoryFixture implements Closeable {
private final File dir;
private Repository repository;
private FileStore fileStore;
private NodeStore nodeStore;
+ private Whiteboard whiteboard;
public RepositoryFixture(File dir) {
this.dir = dir;
@@ -69,6 +75,11 @@ public class RepositoryFixture implement
return nodeStore;
}
+ public AsyncIndexUpdate getAsyncIndexUpdate(String laneName) {
+ return (AsyncIndexUpdate) getService(whiteboard, Runnable.class,
+ (runnable) -> runnable instanceof AsyncIndexUpdate &&
laneName.equals(((AsyncIndexUpdate)runnable).getName()));
+ }
+
@Override
public void close() throws IOException {
if (repository instanceof JackrabbitRepository) {
@@ -80,6 +91,8 @@ public class RepositoryFixture implement
fileStore.close();
fileStore = null;
}
+
+ whiteboard = null;
}
public File getDir() {
@@ -88,10 +101,14 @@ public class RepositoryFixture implement
private Repository createRepository() throws IOException {
Oak oak = new Oak(getNodeStore());
+
+ oak.withAsyncIndexing("async", 3600); //Effectively disable async
indexing
configureLuceneProvider(oak);
Jcr jcr = new Jcr(oak);
- return jcr.createRepository();
+ Repository repository = jcr.createRepository();
+ whiteboard = oak.getWhiteboard();
+ return repository;
}
private void configureLuceneProvider(Oak oak) throws IOException {
