Author: chetanm
Date: Wed Sep 27 05:19:44 2017
New Revision: 1809803
URL: http://svn.apache.org/viewvc?rev=1809803&view=rev
Log:
OAK-6718 - Configure RAM buffer size used by IndexWriter with oak-run indexing
Introduce a new LuceneIndexWriterConfig which provides a way to pass in custom
config option to Lucene IndexWriter
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/LuceneIndexWriterConfig.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterFactory.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/AggregateTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiplexingLucenePropertyIndexTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReaderFactoryTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/LuceneIndexerProvider.java
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
Wed Sep 27 05:19:44 2017
@@ -34,6 +34,7 @@ import org.apache.jackrabbit.oak.plugins
import
org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexWriterFactory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LuceneDocumentHolder;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.DefaultIndexWriterFactory;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterConfig;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterFactory;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.commit.CommitContext;
@@ -68,6 +69,7 @@ public class LuceneIndexEditorProvider i
private GarbageCollectableBlobStore blobStore;
private IndexingQueue indexingQueue;
private boolean nrtIndexingEnabled;
+ private LuceneIndexWriterConfig writerConfig = new
LuceneIndexWriterConfig();
/**
* Number of indexed Lucene document that can be held in memory
@@ -179,7 +181,7 @@ public class LuceneIndexEditorProvider i
}
if (writerFactory == null) {
- writerFactory = new
DefaultIndexWriterFactory(mountInfoProvider,
newDirectoryFactory(blobDeletionCallback));
+ writerFactory = new
DefaultIndexWriterFactory(mountInfoProvider,
newDirectoryFactory(blobDeletionCallback), writerConfig);
}
LuceneIndexEditorContext context = new
LuceneIndexEditorContext(root, definition, indexDefinition, callback,
@@ -227,6 +229,10 @@ public class LuceneIndexEditorProvider i
this.nrtIndexingEnabled = indexingQueue != null;
}
+ public void setWriterConfig(LuceneIndexWriterConfig writerConfig) {
+ this.writerConfig = writerConfig;
+ }
+
GarbageCollectableBlobStore getBlobStore() {
return blobStore;
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java
Wed Sep 27 05:19:44 2017
@@ -59,6 +59,7 @@ class DefaultIndexWriter implements Luce
private final String dirName;
private final String suggestDirName;
private final boolean reindex;
+ private final LuceneIndexWriterConfig writerConfig;
private IndexWriter writer;
private Directory directory;
private long genAtStart = -1;
@@ -66,13 +67,14 @@ class DefaultIndexWriter implements Luce
public DefaultIndexWriter(IndexDefinition definition, NodeBuilder
definitionBuilder,
DirectoryFactory directoryFactory, String
dirName, String suggestDirName,
- boolean reindex) {
+ boolean reindex, LuceneIndexWriterConfig
writerConfig) {
this.definition = definition;
this.definitionBuilder = definitionBuilder;
this.directoryFactory = directoryFactory;
this.dirName = dirName;
this.suggestDirName = suggestDirName;
this.reindex = reindex;
+ this.writerConfig = writerConfig;
}
@Override
@@ -142,7 +144,7 @@ class DefaultIndexWriter implements Luce
if (writer == null) {
final long start = PERF_LOGGER.start();
directory = directoryFactory.newInstance(definition,
definitionBuilder, dirName, reindex);
- IndexWriterConfig config = getIndexWriterConfig(definition,
directoryFactory.remoteDirectory());
+ IndexWriterConfig config = getIndexWriterConfig(definition,
directoryFactory.remoteDirectory(), writerConfig);
config.setMergePolicy(definition.getMergePolicy());
writer = new IndexWriter(directory, config);
genAtStart = getLatestGeneration(directory);
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterFactory.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterFactory.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterFactory.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterFactory.java
Wed Sep 27 05:19:44 2017
@@ -19,14 +19,10 @@
package org.apache.jackrabbit.oak.plugins.index.lucene.writer;
-import javax.annotation.Nullable;
-import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
-import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.DefaultDirectoryFactory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryFactory;
-import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -35,16 +31,13 @@ import static com.google.common.base.Pre
public class DefaultIndexWriterFactory implements LuceneIndexWriterFactory {
private final MountInfoProvider mountInfoProvider;
private final DirectoryFactory directoryFactory;
+ private final LuceneIndexWriterConfig writerConfig;
public DefaultIndexWriterFactory(MountInfoProvider mountInfoProvider,
- @Nullable IndexCopier indexCopier, @Nullable
GarbageCollectableBlobStore blobStore) {
- this(mountInfoProvider, new DefaultDirectoryFactory(indexCopier,
blobStore));
- }
-
- public DefaultIndexWriterFactory(MountInfoProvider mountInfoProvider,
- DirectoryFactory directoryFactory) {
+ DirectoryFactory directoryFactory,
LuceneIndexWriterConfig writerConfig) {
this.mountInfoProvider = checkNotNull(mountInfoProvider);
this.directoryFactory = checkNotNull(directoryFactory);
+ this.writerConfig = checkNotNull(writerConfig);
}
@Override
@@ -52,10 +45,10 @@ public class DefaultIndexWriterFactory i
NodeBuilder definitionBuilder,
boolean reindex) {
if (mountInfoProvider.hasNonDefaultMounts()){
return new MultiplexingIndexWriter(directoryFactory,
mountInfoProvider, definition,
- definitionBuilder, reindex);
+ definitionBuilder, reindex, writerConfig);
}
return new DefaultIndexWriter(definition, definitionBuilder,
directoryFactory,
LuceneIndexConstants.INDEX_DATA_CHILD_NAME,
- LuceneIndexConstants.SUGGEST_DATA_CHILD_NAME, reindex);
+ LuceneIndexConstants.SUGGEST_DATA_CHILD_NAME, reindex,
writerConfig);
}
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/IndexWriterUtils.java
Wed Sep 27 05:19:44 2017
@@ -37,7 +37,12 @@ import static org.apache.jackrabbit.oak.
public class IndexWriterUtils {
- public static IndexWriterConfig getIndexWriterConfig(IndexDefinition
definition, boolean remoteDir) {
+ public static IndexWriterConfig getIndexWriterConfig(IndexDefinition
definition, boolean remoteDir){
+ return getIndexWriterConfig(definition, remoteDir, new
LuceneIndexWriterConfig());
+ }
+
+ public static IndexWriterConfig getIndexWriterConfig(IndexDefinition
definition, boolean remoteDir,
+
LuceneIndexWriterConfig writerConfig) {
// FIXME: Hack needed to make Lucene work in an OSGi environment
Thread thread = Thread.currentThread();
ClassLoader loader = thread.getContextClassLoader();
@@ -51,13 +56,13 @@ public class IndexWriterUtils {
}
Analyzer analyzer = new
PerFieldAnalyzerWrapper(definitionAnalyzer, analyzers);
IndexWriterConfig config = new IndexWriterConfig(VERSION,
analyzer);
-// config.setMergePolicy(definition.getMergePolicy());
if (remoteDir) {
config.setMergeScheduler(new SerialMergeScheduler());
}
if (definition.getCodec() != null) {
config.setCodec(definition.getCodec());
}
+ config.setRAMBufferSizeMB(writerConfig.getRamBufferSizeMB());
return config;
} finally {
thread.setContextClassLoader(loader);
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/LuceneIndexWriterConfig.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/LuceneIndexWriterConfig.java?rev=1809803&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/LuceneIndexWriterConfig.java
(added)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/LuceneIndexWriterConfig.java
Wed Sep 27 05:19:44 2017
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.writer;
+
+import org.apache.lucene.index.IndexWriterConfig;
+
+public class LuceneIndexWriterConfig {
+ private final double ramBufferSizeMB;
+
+ public LuceneIndexWriterConfig() {
+ this(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
+ }
+
+ public LuceneIndexWriterConfig(double ramBufferSizeMB) {
+ this.ramBufferSizeMB = ramBufferSizeMB;
+ }
+
+ public double getRamBufferSizeMB() {
+ return ramBufferSizeMB;
+ }
+}
Propchange:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/LuceneIndexWriterConfig.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java
Wed Sep 27 05:19:44 2017
@@ -37,17 +37,19 @@ class MultiplexingIndexWriter implements
private final IndexDefinition definition;
private final NodeBuilder definitionBuilder;
private final boolean reindex;
+ private final LuceneIndexWriterConfig writerConfig;
private final Map<Mount, DefaultIndexWriter> writers = Maps.newHashMap();
public MultiplexingIndexWriter(DirectoryFactory directoryFactory,
MountInfoProvider mountInfoProvider,
IndexDefinition definition, NodeBuilder
definitionBuilder,
- boolean reindex) {
+ boolean reindex, LuceneIndexWriterConfig
writerConfig) {
this.mountInfoProvider = mountInfoProvider;
this.definition = definition;
this.definitionBuilder = definitionBuilder;
this.reindex = reindex;
this.directoryFactory = directoryFactory;
+ this.writerConfig = writerConfig;
}
@Override
@@ -97,6 +99,6 @@ class MultiplexingIndexWriter implements
String dirName = MultiplexersLucene.getIndexDirName(m);
String suggestDirName = MultiplexersLucene.getSuggestDirName(m);
return new DefaultIndexWriter(definition, definitionBuilder,
directoryFactory, dirName,
- suggestDirName, reindex);
+ suggestDirName, reindex, writerConfig);
}
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/AggregateTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/AggregateTest.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/AggregateTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/AggregateTest.java
Wed Sep 27 05:19:44 2017
@@ -39,9 +39,12 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.plugins.index.lucene.Aggregate.NodeInclude;
import
org.apache.jackrabbit.oak.plugins.index.lucene.Aggregate.NodeIncludeResult;
import
org.apache.jackrabbit.oak.plugins.index.lucene.Aggregate.PropertyIncludeResult;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.DefaultDirectoryFactory;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryFactory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.DefaultIndexWriterFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterConfig;
import org.apache.jackrabbit.oak.spi.mount.Mounts;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -420,7 +423,9 @@ public class AggregateTest {
LuceneDocumentMaker maker = new LuceneDocumentMaker(defn,
defn.getApplicableIndexingRule("nt:folder"), "/bar");
Document doc = maker.makeDocument(nb.getNodeState());
- DefaultIndexWriterFactory writerFactory = new
DefaultIndexWriterFactory(Mounts.defaultMountInfoProvider(), null, null);
+ DirectoryFactory directoryFactory = new DefaultDirectoryFactory(null,
null);
+ DefaultIndexWriterFactory writerFactory = new
DefaultIndexWriterFactory(Mounts.defaultMountInfoProvider(),
+ directoryFactory, new LuceneIndexWriterConfig());
LuceneIndexWriter writer = writerFactory.newInstance(defn,
EMPTY_NODE.builder(), false);
writer.updateDocument("/bar", doc);
writer.close(100);
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiplexingLucenePropertyIndexTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiplexingLucenePropertyIndexTest.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiplexingLucenePropertyIndexTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiplexingLucenePropertyIndexTest.java
Wed Sep 27 05:19:44 2017
@@ -38,11 +38,14 @@ import org.apache.jackrabbit.oak.api.Con
import org.apache.jackrabbit.oak.api.Result;
import org.apache.jackrabbit.oak.api.Tree;
import org.apache.jackrabbit.oak.commons.PathUtils;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.DefaultDirectoryFactory;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryFactory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader;
import
org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.DefaultIndexWriterFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterConfig;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterFactory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.MultiplexersLucene;
import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
@@ -129,7 +132,8 @@ public class MultiplexingLucenePropertyI
IndexDefinition defn = new IndexDefinition(initialContent,
defnBuilder.getNodeState(), "/foo");
//1. Have 2 reader created by writes in 2 diff mounts
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, null);
+ DirectoryFactory directoryFactory = new DefaultDirectoryFactory(null,
null);
+ LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
directoryFactory, new LuceneIndexWriterConfig());
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
writer.updateDocument("/content/en", newDoc("/content/en"));
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReaderFactoryTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReaderFactoryTest.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReaderFactoryTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReaderFactoryTest.java
Wed Sep 27 05:19:44 2017
@@ -29,8 +29,11 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.DefaultDirectoryFactory;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryFactory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.DefaultIndexWriterFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterConfig;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterFactory;
import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
import org.apache.jackrabbit.oak.spi.mount.Mounts;
@@ -57,6 +60,7 @@ public class DefaultIndexReaderFactoryTe
private IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState(), "/foo");
private MountInfoProvider mip = Mounts.newBuilder()
.mount("foo", "/libs", "/apps").build();
+ private LuceneIndexWriterConfig writerConfig = new
LuceneIndexWriterConfig();
@Test
public void emptyDir() throws Exception{
@@ -67,7 +71,7 @@ public class DefaultIndexReaderFactoryTe
@Test
public void indexDir() throws Exception{
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, null);
+ LuceneIndexWriterFactory factory = newDirectoryFactory();
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
writer.updateDocument("/content/en", newDoc("/content/en"));
@@ -104,7 +108,8 @@ public class DefaultIndexReaderFactoryTe
.createCachingFDS(folder.newFolder().getAbsolutePath(),
folder.newFolder().getAbsolutePath());
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, new DataStoreBlobStore(ds));
+ DirectoryFactory directoryFactory = new DefaultDirectoryFactory(null,
new DataStoreBlobStore(ds));
+ LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
directoryFactory, writerConfig);
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
writer.updateDocument("/content/en", newDoc("/content/en"));
@@ -136,7 +141,7 @@ public class DefaultIndexReaderFactoryTe
@Test
public void suggesterDir() throws Exception{
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, null);
+ LuceneIndexWriterFactory factory = newDirectoryFactory();
enabledSuggestorForSomeProp();
defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
@@ -156,7 +161,7 @@ public class DefaultIndexReaderFactoryTe
@Test
public void multipleReaders() throws Exception{
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, null);
+ LuceneIndexWriterFactory factory = newDirectoryFactory();
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
writer.updateDocument("/content/en", newDoc("/content/en"));
@@ -170,7 +175,7 @@ public class DefaultIndexReaderFactoryTe
@Test
public void multipleReaders_SingleSuggester() throws Exception{
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, null);
+ LuceneIndexWriterFactory factory = newDirectoryFactory();
enabledSuggestorForSomeProp();
defn = new IndexDefinition(root, builder.getNodeState(), "/foo");
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
@@ -199,4 +204,9 @@ public class DefaultIndexReaderFactoryTe
prop.setProperty("name", "foo");
prop.setProperty(LuceneIndexConstants.PROP_USE_IN_SUGGEST, true);
}
+
+ private LuceneIndexWriterFactory newDirectoryFactory(){
+ DirectoryFactory directoryFactory = new DefaultDirectoryFactory(null,
null);
+ return new DefaultIndexWriterFactory(mip, directoryFactory,
writerConfig);
+ }
}
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterTest.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterTest.java
Wed Sep 27 05:19:44 2017
@@ -40,6 +40,7 @@ import static org.apache.jackrabbit.oak.
import static
org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
import static org.apache.jackrabbit.oak.InitialContent.INITIAL_CONTENT;
import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
@@ -53,27 +54,26 @@ public class DefaultIndexWriterTest {
private NodeBuilder builder = EMPTY_NODE.builder();
+ private LuceneIndexWriterConfig writerConfig = new
LuceneIndexWriterConfig();
+
@Test
public void lazyInit() throws Exception {
IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState(), "/foo");
- DefaultIndexWriter writer = new DefaultIndexWriter(defn, builder, null,
- INDEX_DATA_CHILD_NAME, SUGGEST_DATA_CHILD_NAME, false);
+ DefaultIndexWriter writer = createWriter(defn, false);
assertFalse(writer.close(0));
}
@Test
public void writeInitializedUponReindex() throws Exception {
IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState(), "/foo");
- DefaultIndexWriter writer = new DefaultIndexWriter(defn, builder,
- new DefaultDirectoryFactory(null, null),
INDEX_DATA_CHILD_NAME, SUGGEST_DATA_CHILD_NAME, true);
+ DefaultIndexWriter writer = createWriter(defn, true);
assertTrue(writer.close(0));
}
@Test
public void indexUpdated() throws Exception {
IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState(), "/foo");
- DefaultIndexWriter writer = new DefaultIndexWriter(defn, builder,
- new DefaultDirectoryFactory(null, null),
INDEX_DATA_CHILD_NAME, SUGGEST_DATA_CHILD_NAME, false);
+ DefaultIndexWriter writer = createWriter(defn, false);
Document document = new Document();
document.add(newPathField("/a/b"));
@@ -86,8 +86,7 @@ public class DefaultIndexWriterTest {
@Test
public void indexWriterConfig_Scheduler_Remote() throws Exception{
IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState(), "/foo");
- DefaultIndexWriter writer = new DefaultIndexWriter(defn, builder,
- new DefaultDirectoryFactory(null, null),
INDEX_DATA_CHILD_NAME, SUGGEST_DATA_CHILD_NAME, true);
+ DefaultIndexWriter writer = createWriter(defn, true);
IndexWriter w = writer.getWriter();
assertThat(w.getConfig().getMergeScheduler(),
instanceOf(SerialMergeScheduler.class));
@@ -98,9 +97,28 @@ public class DefaultIndexWriterTest {
FSDirectoryFactory fsdir = new FSDirectoryFactory(folder.getRoot());
IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState(), "/foo");
DefaultIndexWriter writer = new DefaultIndexWriter(defn, builder,
- fsdir, INDEX_DATA_CHILD_NAME, SUGGEST_DATA_CHILD_NAME, true);
+ fsdir, INDEX_DATA_CHILD_NAME, SUGGEST_DATA_CHILD_NAME, true,
writerConfig);
IndexWriter w = writer.getWriter();
assertThat(w.getConfig().getMergeScheduler(),
instanceOf(ConcurrentMergeScheduler.class));
}
+
+ @Test
+ public void configRAMSize() throws Exception{
+ writerConfig = new LuceneIndexWriterConfig(42);
+
+ IndexDefinition defn = new IndexDefinition(root,
builder.getNodeState(), "/foo");
+ DefaultIndexWriter writer = createWriter(defn, true);
+
+ IndexWriter w = writer.getWriter();
+ assertEquals(w.getConfig().getRAMBufferSizeMB(), 42, 0);
+ }
+
+ private DefaultIndexWriter createWriter(IndexDefinition defn, boolean
reindex) {
+ return new DefaultIndexWriter(defn, builder,
+ new DefaultDirectoryFactory(null, null), INDEX_DATA_CHILD_NAME,
+ SUGGEST_DATA_CHILD_NAME, reindex, writerConfig);
+ }
+
+
}
\ No newline at end of file
Modified:
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java
(original)
+++
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java
Wed Sep 27 05:19:44 2017
@@ -29,6 +29,8 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.DefaultDirectoryFactory;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.directory.OakDirectory;
import org.apache.jackrabbit.oak.spi.mount.Mount;
import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
@@ -65,6 +67,7 @@ public class MultiplexingIndexWriterTest
private Mount fooMount;
private Mount defaultMount;
+ private LuceneIndexWriterConfig writerConfig = new
LuceneIndexWriterConfig();
@Before
public void setUp(){
@@ -73,15 +76,14 @@ public class MultiplexingIndexWriterTest
@Test
public void defaultWriterWithNoMounts() throws Exception{
- LuceneIndexWriterFactory factory = new
DefaultIndexWriterFactory(Mounts.defaultMountInfoProvider(), null,
- null);
+ LuceneIndexWriterFactory factory =
newDirectoryFactory(Mounts.defaultMountInfoProvider());
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
assertThat(writer, instanceOf(DefaultIndexWriter.class));
}
@Test
public void closeWithoutChange() throws Exception{
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, null);
+ LuceneIndexWriterFactory factory = newDirectoryFactory();
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
assertFalse(writer.close(0));
assertEquals(0, Iterables.size(getIndexDirNodes()));
@@ -89,7 +91,7 @@ public class MultiplexingIndexWriterTest
@Test
public void writesInDefaultMount() throws Exception{
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, null);
+ LuceneIndexWriterFactory factory = newDirectoryFactory();
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
//1. Add entry in foo mount
@@ -115,7 +117,8 @@ public class MultiplexingIndexWriterTest
.createCachingFDS(folder.newFolder().getAbsolutePath(),
folder.newFolder().getAbsolutePath());
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, new DataStoreBlobStore(ds));
+ DirectoryFactory directoryFactory = new DefaultDirectoryFactory(null,
new DataStoreBlobStore(ds));
+ LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
directoryFactory, writerConfig);
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
//1. Add entry in foo mount
@@ -137,7 +140,7 @@ public class MultiplexingIndexWriterTest
@Test
public void deletes() throws Exception{
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, null);
+ LuceneIndexWriterFactory factory = newDirectoryFactory();
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
writer.updateDocument("/libs/config", newDoc("/libs/config"));
@@ -169,7 +172,7 @@ public class MultiplexingIndexWriterTest
mip = Mounts.newBuilder()
.mount("foo", "/content/remote").build();
initializeMounts();
- LuceneIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
null, null);
+ LuceneIndexWriterFactory factory = newDirectoryFactory();
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
writer.updateDocument("/content/remote/a",
newDoc("/content/remote/a"));
@@ -215,4 +218,13 @@ public class MultiplexingIndexWriterTest
return MultiplexersLucene.getIndexDirName(m);
}
+ private LuceneIndexWriterFactory newDirectoryFactory(){
+ return newDirectoryFactory(mip);
+ }
+
+ private LuceneIndexWriterFactory newDirectoryFactory(MountInfoProvider
mountInfoProvider){
+ DirectoryFactory directoryFactory = new DefaultDirectoryFactory(null,
null);
+ return new DefaultIndexWriterFactory(mountInfoProvider,
directoryFactory, writerConfig);
+ }
+
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/LuceneIndexerProvider.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/LuceneIndexerProvider.java?rev=1809803&r1=1809802&r2=1809803&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/LuceneIndexerProvider.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/LuceneIndexerProvider.java
Wed Sep 27 05:19:44 2017
@@ -34,6 +34,7 @@ import org.apache.jackrabbit.oak.plugins
import
org.apache.jackrabbit.oak.plugins.index.lucene.directory.FSDirectoryFactory;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.DefaultIndexWriterFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
+import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterConfig;
import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterFactory;
import
org.apache.jackrabbit.oak.plugins.index.progress.IndexingProgressReporter;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -52,7 +53,8 @@ public class LuceneIndexerProvider imple
public LuceneIndexerProvider(IndexHelper indexHelper, IndexerSupport
indexerSupport) throws IOException {
this.indexHelper = indexHelper;
this.dirFactory = new
FSDirectoryFactory(indexerSupport.getLocalIndexDir());
- this.indexWriterFactory = new
DefaultIndexWriterFactory(indexHelper.getMountInfoProvider(), dirFactory);
+ this.indexWriterFactory = new
DefaultIndexWriterFactory(indexHelper.getMountInfoProvider(),
+ dirFactory, new LuceneIndexWriterConfig());
}
@Override
