This is an automated email from the ASF dual-hosted git repository.
xbli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4cb27af1ea add feature flag for recent TextIndex optimizations (#13577)
4cb27af1ea is described below
commit 4cb27af1ea1d4f4a228d2127cb8468189b304cf7
Author: Xiaobing <[email protected]>
AuthorDate: Wed Jul 10 17:21:29 2024 -0700
add feature flag for recent TextIndex optimizations (#13577)
* add feature flag for recent TextIndex optimizations
* expand the reuseParams test data
* add cache size config for NRT cache
---
.../invertedindex/RealtimeLuceneTextIndex.java | 10 ++--
.../creator/impl/text/LuceneTextIndexCreator.java | 32 +++++++------
.../converter/RealtimeSegmentConverterTest.java | 18 +++++---
.../invertedindex/LuceneMutableTextIndexTest.java | 2 +-
.../NativeAndLuceneMutableTextIndexTest.java | 2 +-
.../segment/store/FilePerIndexDirectoryTest.java | 17 +++----
.../store/SingleFileIndexDirectoryTest.java | 4 +-
.../pinot/segment/spi/index/TextIndexConfig.java | 54 ++++++++++++++++++----
8 files changed, 92 insertions(+), 47 deletions(-)
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeLuceneTextIndex.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeLuceneTextIndex.java
index d858655843..0175bc5109 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeLuceneTextIndex.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/RealtimeLuceneTextIndex.java
@@ -55,6 +55,7 @@ public class RealtimeLuceneTextIndex implements
MutableTextIndex {
private Analyzer _analyzer;
private final String _column;
private final String _segmentName;
+ private final boolean _reuseMutableIndex;
private boolean _enablePrefixSuffixMatchingInPhraseQueries = false;
private final RealtimeLuceneRefreshListener _refreshListener;
@@ -90,6 +91,7 @@ public class RealtimeLuceneTextIndex implements
MutableTextIndex {
_searcherManager.addListener(_refreshListener);
_analyzer = _indexCreator.getIndexWriter().getConfig().getAnalyzer();
_enablePrefixSuffixMatchingInPhraseQueries =
config.isEnablePrefixSuffixMatchingInPhraseQueries();
+ _reuseMutableIndex = config.isReuseMutableIndex();
} catch (Exception e) {
LOGGER.error("Failed to instantiate realtime Lucene index reader for
column {}, exception {}", column,
e.getMessage());
@@ -152,8 +154,7 @@ public class RealtimeLuceneTextIndex implements
MutableTextIndex {
}
}
};
- Future<MutableRoaringBitmap> searchFuture =
- SEARCHER_POOL.getExecutorService().submit(searchCallable);
+ Future<MutableRoaringBitmap> searchFuture =
SEARCHER_POOL.getExecutorService().submit(searchCallable);
try {
return searchFuture.get();
} catch (InterruptedException e) {
@@ -163,7 +164,7 @@ public class RealtimeLuceneTextIndex implements
MutableTextIndex {
throw new RuntimeException("TEXT_MATCH query interrupted while querying
the consuming segment");
} catch (Exception e) {
LOGGER.error("Failed while searching the realtime text index for segment
{}, column {}, search query {},"
- + " exception {}", _segmentName, _column, searchQuery,
e.getMessage());
+ + " exception {}", _segmentName, _column, searchQuery,
e.getMessage());
throw new RuntimeException(e);
}
}
@@ -190,6 +191,9 @@ public class RealtimeLuceneTextIndex implements
MutableTextIndex {
@Override
public void commit() {
+ if (!_reuseMutableIndex) {
+ return;
+ }
try {
_indexCreator.getIndexWriter().commit();
} catch (Exception e) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
index 36b02b7d20..fc2a9ae151 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/text/LuceneTextIndexCreator.java
@@ -113,9 +113,6 @@ public class LuceneTextIndexCreator extends
AbstractTextIndexCreator {
_textColumn = column;
_commitOnClose = commit;
- // to reuse the mutable index, it must be (1) not the realtime index, i.e.
commit is set to false
- // and (2) happens during realtime segment conversion
- _reuseMutableIndex = commit && realtimeConversion;
String luceneAnalyzerClass = config.getLuceneAnalyzerClass();
try {
// segment generation is always in V1 and later we convert (as part of
post creation processing)
@@ -135,22 +132,25 @@ public class LuceneTextIndexCreator extends
AbstractTextIndexCreator {
indexWriterConfig.setCommitOnClose(commit);
indexWriterConfig.setUseCompoundFile(config.isLuceneUseCompoundFile());
- // For the realtime segment, prevent background merging. The realtime
segment will call .commit()
- // on the IndexWriter when segment conversion occurs. By default, Lucene
will sometimes choose to
- // merge segments in the background, which is problematic because the
lucene index directory's
- // contents is copied to create the immutable segment. If a background
merge occurs during this
- // copy, a FileNotFoundException will be triggered and segment build
will fail.
+ // For the realtime segment, to reuse mutable index, we should set the
two write configs below.
+ // The realtime segment will call .commit() on the IndexWriter when
segment conversion occurs.
+ // By default, Lucene will sometimes choose to merge segments in the
background, which is problematic because
+ // the lucene index directory's contents is copied to create the
immutable segment. If a background merge
+ // occurs during this copy, a FileNotFoundException will be triggered
and segment build will fail.
//
// Also, for the realtime segment, we set the OpenMode to CREATE to
ensure that any existing artifacts
// will be overwritten. This is necessary because the realtime segment
can be created multiple times
// during a server crash and restart scenario. If the existing artifacts
are appended to, the realtime
// query results will be accurate, but after segment conversion the
mapping file generated will be loaded
// for only the first numDocs lucene docIds, which can cause
IndexOutOfBounds errors.
- if (!_commitOnClose) {
+ if (!_commitOnClose && config.isReuseMutableIndex()) {
indexWriterConfig.setMergeScheduler(NoMergeScheduler.INSTANCE);
indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
}
+ // to reuse the mutable index, it must be (1) not the realtime index,
i.e. commit is set to true
+ // and (2) happens during realtime segment conversion
+ _reuseMutableIndex = config.isReuseMutableIndex() && commit &&
realtimeConversion;
if (_reuseMutableIndex) {
LOGGER.info("Reusing the realtime lucene index for segment {} and
column {}", segmentIndexDir, column);
indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
@@ -158,15 +158,17 @@ public class LuceneTextIndexCreator extends
AbstractTextIndexCreator {
return;
}
- if (_commitOnClose) {
- _indexDirectory = FSDirectory.open(_indexFile.toPath());
- } else {
+ if (!_commitOnClose &&
config.getLuceneNRTCachingDirectoryMaxBufferSizeMB() > 0) {
// For realtime index, use NRTCachingDirectory to reduce the number of
open files. This buffers the
// flushes triggered by the near real-time refresh and writes them to
disk when the buffer is full,
// reducing the number of small writes.
- _indexDirectory =
- new NRTCachingDirectory(FSDirectory.open(_indexFile.toPath()),
config.getLuceneMaxBufferSizeMB(),
- config.getLuceneMaxBufferSizeMB());
+ int bufSize = config.getLuceneNRTCachingDirectoryMaxBufferSizeMB();
+ LOGGER.info(
+ "Using NRTCachingDirectory for realtime lucene index for segment
{} and column {} with buffer size: {}MB",
+ segmentIndexDir, column, bufSize);
+ _indexDirectory = new
NRTCachingDirectory(FSDirectory.open(_indexFile.toPath()), bufSize, bufSize);
+ } else {
+ _indexDirectory = FSDirectory.open(_indexFile.toPath());
}
_indexWriter = new IndexWriter(_indexDirectory, indexWriterConfig);
} catch (ReflectiveOperationException e) {
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/RealtimeSegmentConverterTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/RealtimeSegmentConverterTest.java
index c08fe41b0c..390a2d158e 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/RealtimeSegmentConverterTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/converter/RealtimeSegmentConverterTest.java
@@ -447,17 +447,21 @@ public class RealtimeSegmentConverterTest {
@DataProvider
public static Object[][] reuseParams() {
List<Boolean> enabledColumnMajorSegmentBuildParams = Arrays.asList(false,
true);
- String[] sortedColumnParams = new String[]{null, STRING_COLUMN1};
-
- return enabledColumnMajorSegmentBuildParams.stream().flatMap(
- columnMajor -> Arrays.stream(sortedColumnParams).map(sortedColumn
-> new Object[]{columnMajor,
- sortedColumn}))
+ List<String> sortedColumnParams = Arrays.asList(null, STRING_COLUMN1);
+ List<Boolean> reuseMutableIndex = Arrays.asList(true, false);
+ List<Integer> luceneNRTCachingDirectoryMaxBufferSizeMB = Arrays.asList(0,
5);
+
+ return enabledColumnMajorSegmentBuildParams.stream().flatMap(columnMajor
-> sortedColumnParams.stream().flatMap(
+ sortedColumn -> reuseMutableIndex.stream().flatMap(
+ reuseIndex -> luceneNRTCachingDirectoryMaxBufferSizeMB.stream()
+ .map(cacheSize -> new Object[]{columnMajor, sortedColumn,
reuseIndex, cacheSize}))))
.toArray(Object[][]::new);
}
// Test the realtime segment conversion of a table with an index that reuses
mutable index artifacts during conversion
@Test(dataProvider = "reuseParams")
- public void testSegmentBuilderWithReuse(boolean columnMajorSegmentBuilder,
String sortedColumn)
+ public void testSegmentBuilderWithReuse(boolean columnMajorSegmentBuilder,
String sortedColumn,
+ boolean reuseMutableIndex, int luceneNRTCachingDirectoryMaxBufferSizeMB)
throws Exception {
File tmpDir = new File(TMP_DIR, "tmp_" + System.currentTimeMillis());
FieldConfig textIndexFieldConfig =
@@ -477,7 +481,7 @@ public class RealtimeSegmentConverterTest {
IndexingConfig indexingConfig = tableConfig.getIndexingConfig();
TextIndexConfig textIndexConfig =
new TextIndexConfig(false, null, null, false, false,
Collections.emptyList(), Collections.emptyList(), false,
- 500, null, false);
+ 500, null, false, reuseMutableIndex,
luceneNRTCachingDirectoryMaxBufferSizeMB);
RealtimeSegmentConfig.Builder realtimeSegmentConfigBuilder =
new
RealtimeSegmentConfig.Builder().setTableNameWithType(tableNameWithType).setSegmentName(segmentName)
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java
index efa00fe3bf..324cc1fa5d 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/LuceneMutableTextIndexTest.java
@@ -65,7 +65,7 @@ public class LuceneMutableTextIndexTest {
public void setUp()
throws Exception {
TextIndexConfig config =
- new TextIndexConfig(false, null, null, false, false, null, null,
true, 500, null, false);
+ new TextIndexConfig(false, null, null, false, false, null, null, true,
500, null, false, false, 0);
_realtimeLuceneTextIndex =
new RealtimeLuceneTextIndex(TEXT_COLUMN_NAME, INDEX_DIR,
"table__0__1__20240602T0014Z", config);
String[][] documents = getTextData();
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeAndLuceneMutableTextIndexTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeAndLuceneMutableTextIndexTest.java
index 6c1c664694..932c851be0 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeAndLuceneMutableTextIndexTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/realtime/impl/invertedindex/NativeAndLuceneMutableTextIndexTest.java
@@ -75,7 +75,7 @@ public class NativeAndLuceneMutableTextIndexTest {
throws Exception {
ServerMetrics.register(mock(ServerMetrics.class));
TextIndexConfig config =
- new TextIndexConfig(false, null, null, false, false, null, null, true,
500, null, false);
+ new TextIndexConfig(false, null, null, false, false, null, null, true,
500, null, false, false, 0);
_realtimeLuceneTextIndex =
new RealtimeLuceneTextIndex(TEXT_COLUMN_NAME, INDEX_DIR,
"table__0__1__20240602T0014Z", config);
_nativeMutableTextIndex = new NativeMutableTextIndex(TEXT_COLUMN_NAME);
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
index 97e12bc321..65a1621dfc 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/FilePerIndexDirectoryTest.java
@@ -202,7 +202,7 @@ public class FilePerIndexDirectoryTest {
public void testRemoveTextIndices()
throws IOException {
TextIndexConfig config =
- new TextIndexConfig(false, null, null, false, false, null, null,
true, 500, null, false);
+ new TextIndexConfig(false, null, null, false, false, null, null, true,
500, null, false, false, 0);
try (FilePerIndexDirectory fpi = new FilePerIndexDirectory(TEMP_DIR,
_segmentMetadata, ReadMode.mmap);
LuceneTextIndexCreator fooCreator = new LuceneTextIndexCreator("foo",
TEMP_DIR, true, false, null, null,
config);
@@ -266,7 +266,7 @@ public class FilePerIndexDirectoryTest {
public void testGetColumnIndices()
throws IOException {
TextIndexConfig config =
- new TextIndexConfig(false, null, null, false, false, null, null,
true, 500, null, false);
+ new TextIndexConfig(false, null, null, false, false, null, null, true,
500, null, false, false, 0);
// Write sth to buffers and flush them to index files on disk
try (FilePerIndexDirectory fpi = new FilePerIndexDirectory(TEMP_DIR,
_segmentMetadata, ReadMode.mmap);
LuceneTextIndexCreator fooCreator = new LuceneTextIndexCreator("foo",
TEMP_DIR, true, false, null, null,
@@ -293,17 +293,15 @@ public class FilePerIndexDirectoryTest {
}
// Need segmentMetadata to tell the full set of columns in this segment.
- when(_segmentMetadata.getAllColumns())
- .thenReturn(new TreeSet<>(Arrays.asList("col1", "col2", "col3",
"col4", "col5", "foo", "bar")));
+ when(_segmentMetadata.getAllColumns()).thenReturn(
+ new TreeSet<>(Arrays.asList("col1", "col2", "col3", "col4", "col5",
"foo", "bar")));
try (FilePerIndexDirectory fpi = new FilePerIndexDirectory(TEMP_DIR,
_segmentMetadata, ReadMode.mmap)) {
- assertEquals(fpi.getColumnsWithIndex(StandardIndexes.forward()),
- new HashSet<>(Arrays.asList("col1", "col3")));
+ assertEquals(fpi.getColumnsWithIndex(StandardIndexes.forward()), new
HashSet<>(Arrays.asList("col1", "col3")));
assertEquals(fpi.getColumnsWithIndex(StandardIndexes.dictionary()),
new HashSet<>(Collections.singletonList("col2")));
assertEquals(fpi.getColumnsWithIndex(StandardIndexes.inverted()),
new HashSet<>(Collections.singletonList("col4")));
- assertEquals(fpi.getColumnsWithIndex(StandardIndexes.h3()),
- new HashSet<>(Collections.singletonList("col5")));
+ assertEquals(fpi.getColumnsWithIndex(StandardIndexes.h3()), new
HashSet<>(Collections.singletonList("col5")));
assertEquals(fpi.getColumnsWithIndex(StandardIndexes.text()), new
HashSet<>(Arrays.asList("foo", "bar")));
fpi.removeIndex("col1", StandardIndexes.forward());
@@ -318,8 +316,7 @@ public class FilePerIndexDirectoryTest {
assertEquals(fpi.getColumnsWithIndex(StandardIndexes.inverted()),
new HashSet<>(Collections.singletonList("col4")));
assertEquals(fpi.getColumnsWithIndex(StandardIndexes.h3()), new
HashSet<>(Collections.emptySet()));
- assertEquals(fpi.getColumnsWithIndex(StandardIndexes.text()),
- new HashSet<>(Collections.singletonList("bar")));
+ assertEquals(fpi.getColumnsWithIndex(StandardIndexes.text()), new
HashSet<>(Collections.singletonList("bar")));
}
}
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
index b962550e3b..7e9933d777 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/store/SingleFileIndexDirectoryTest.java
@@ -235,7 +235,7 @@ public class SingleFileIndexDirectoryTest {
public void testRemoveTextIndices()
throws IOException, ConfigurationException {
TextIndexConfig config =
- new TextIndexConfig(false, null, null, false, false, null, null,
true, 500, null, false);
+ new TextIndexConfig(false, null, null, false, false, null, null, true,
500, null, false, false, 0);
try (SingleFileIndexDirectory sfd = new SingleFileIndexDirectory(TEMP_DIR,
_segmentMetadata, ReadMode.mmap);
LuceneTextIndexCreator fooCreator = new LuceneTextIndexCreator("foo",
TEMP_DIR, true, false, null, null,
config);
@@ -343,7 +343,7 @@ public class SingleFileIndexDirectoryTest {
public void testGetColumnIndices()
throws Exception {
TextIndexConfig config =
- new TextIndexConfig(false, null, null, false, false, null, null,
true, 500, null, false);
+ new TextIndexConfig(false, null, null, false, false, null, null, true,
500, null, false, false, 0);
try (SingleFileIndexDirectory sfd = new SingleFileIndexDirectory(TEMP_DIR,
_segmentMetadata, ReadMode.mmap);
LuceneTextIndexCreator fooCreator = new LuceneTextIndexCreator("foo",
TEMP_DIR, true, false, null, null,
config);
diff --git
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
index 6c400a16db..522fb7bbf2 100644
---
a/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
+++
b/pinot-segment-spi/src/main/java/org/apache/pinot/segment/spi/index/TextIndexConfig.java
@@ -35,10 +35,12 @@ import org.apache.pinot.spi.config.table.IndexConfig;
public class TextIndexConfig extends IndexConfig {
private static final int LUCENE_INDEX_DEFAULT_MAX_BUFFER_SIZE_MB = 500;
private static final boolean LUCENE_INDEX_DEFAULT_USE_COMPOUND_FILE = true;
+ private static final boolean
LUCENE_INDEX_ENABLE_PREFIX_SUFFIX_MATCH_IN_PHRASE_SEARCH = false;
+ private static final boolean LUCENE_INDEX_REUSE_MUTABLE_INDEX = false;
+ private static final int
LUCENE_INDEX_NRT_CACHING_DIRECTORY_MAX_BUFFER_SIZE_MB = 0;
public static final TextIndexConfig DISABLED =
new TextIndexConfig(true, null, null, false, false,
Collections.emptyList(), Collections.emptyList(), false,
- LUCENE_INDEX_DEFAULT_MAX_BUFFER_SIZE_MB, null, false);
- private static final boolean
LUCENE_INDEX_ENABLE_PREFIX_SUFFIX_MATCH_IN_PHRASE_SEARCH = false;
+ LUCENE_INDEX_DEFAULT_MAX_BUFFER_SIZE_MB, null, false, false, 0);
private final FSTType _fstType;
@Nullable
private final Object _rawValueForTextIndex;
@@ -50,6 +52,8 @@ public class TextIndexConfig extends IndexConfig {
private final int _luceneMaxBufferSizeMB;
private final String _luceneAnalyzerClass;
private final boolean _enablePrefixSuffixMatchingInPhraseQueries;
+ private final boolean _reuseMutableIndex;
+ private final int _luceneNRTCachingDirectoryMaxBufferSizeMB;
@JsonCreator
public TextIndexConfig(@JsonProperty("disabled") Boolean disabled,
@JsonProperty("fst") FSTType fstType,
@@ -61,7 +65,9 @@ public class TextIndexConfig extends IndexConfig {
@JsonProperty("luceneUseCompoundFile") Boolean luceneUseCompoundFile,
@JsonProperty("luceneMaxBufferSizeMB") Integer luceneMaxBufferSizeMB,
@JsonProperty("luceneAnalyzerClass") String luceneAnalyzerClass,
- @JsonProperty("enablePrefixSuffixMatchingInPhraseQueries") Boolean
enablePrefixSuffixMatchingInPhraseQueries) {
+ @JsonProperty("enablePrefixSuffixMatchingInPhraseQueries") Boolean
enablePrefixSuffixMatchingInPhraseQueries,
+ @JsonProperty("reuseMutableIndex") Boolean reuseMutableIndex,
+ @JsonProperty("luceneNRTCachingDirectoryMaxBufferSizeMB") Integer
luceneNRTCachingDirectoryMaxBufferSizeMB) {
super(disabled);
_fstType = fstType;
_rawValueForTextIndex = rawValueForTextIndex;
@@ -78,6 +84,10 @@ public class TextIndexConfig extends IndexConfig {
_enablePrefixSuffixMatchingInPhraseQueries =
enablePrefixSuffixMatchingInPhraseQueries == null ?
LUCENE_INDEX_ENABLE_PREFIX_SUFFIX_MATCH_IN_PHRASE_SEARCH
: enablePrefixSuffixMatchingInPhraseQueries;
+ _reuseMutableIndex = reuseMutableIndex == null ?
LUCENE_INDEX_REUSE_MUTABLE_INDEX : reuseMutableIndex;
+ _luceneNRTCachingDirectoryMaxBufferSizeMB =
+ luceneNRTCachingDirectoryMaxBufferSizeMB == null ?
LUCENE_INDEX_NRT_CACHING_DIRECTORY_MAX_BUFFER_SIZE_MB
+ : luceneNRTCachingDirectoryMaxBufferSizeMB;
}
public FSTType getFstType() {
@@ -141,6 +151,14 @@ public class TextIndexConfig extends IndexConfig {
return _enablePrefixSuffixMatchingInPhraseQueries;
}
+ public boolean isReuseMutableIndex() {
+ return _reuseMutableIndex;
+ }
+
+ public int getLuceneNRTCachingDirectoryMaxBufferSizeMB() {
+ return _luceneNRTCachingDirectoryMaxBufferSizeMB;
+ }
+
public static abstract class AbstractBuilder {
@Nullable
protected FSTType _fstType;
@@ -153,7 +171,10 @@ public class TextIndexConfig extends IndexConfig {
protected boolean _luceneUseCompoundFile =
LUCENE_INDEX_DEFAULT_USE_COMPOUND_FILE;
protected int _luceneMaxBufferSizeMB =
LUCENE_INDEX_DEFAULT_MAX_BUFFER_SIZE_MB;
protected String _luceneAnalyzerClass =
FieldConfig.TEXT_INDEX_DEFAULT_LUCENE_ANALYZER_CLASS;
- protected boolean _enablePrefixSuffixMatchingInPhraseQueries = false;
+ protected boolean _enablePrefixSuffixMatchingInPhraseQueries =
+ LUCENE_INDEX_ENABLE_PREFIX_SUFFIX_MATCH_IN_PHRASE_SEARCH;
+ protected boolean _reuseMutableIndex = LUCENE_INDEX_REUSE_MUTABLE_INDEX;
+ protected int _luceneNRTCachingDirectoryMaxBufferSizeMB =
LUCENE_INDEX_NRT_CACHING_DIRECTORY_MAX_BUFFER_SIZE_MB;
public AbstractBuilder(@Nullable FSTType fstType) {
_fstType = fstType;
@@ -169,12 +190,14 @@ public class TextIndexConfig extends IndexConfig {
_luceneMaxBufferSizeMB = other._luceneMaxBufferSizeMB;
_luceneAnalyzerClass = other._luceneAnalyzerClass;
_enablePrefixSuffixMatchingInPhraseQueries =
other._enablePrefixSuffixMatchingInPhraseQueries;
+ _reuseMutableIndex = other._reuseMutableIndex;
+ _luceneNRTCachingDirectoryMaxBufferSizeMB =
other._luceneNRTCachingDirectoryMaxBufferSizeMB;
}
public TextIndexConfig build() {
return new TextIndexConfig(false, _fstType, _rawValueForTextIndex,
_enableQueryCache, _useANDForMultiTermQueries,
_stopWordsInclude, _stopWordsExclude, _luceneUseCompoundFile,
_luceneMaxBufferSizeMB, _luceneAnalyzerClass,
- _enablePrefixSuffixMatchingInPhraseQueries);
+ _enablePrefixSuffixMatchingInPhraseQueries, _reuseMutableIndex,
_luceneNRTCachingDirectoryMaxBufferSizeMB);
}
public abstract AbstractBuilder withProperties(@Nullable Map<String,
String> textIndexProperties);
@@ -214,6 +237,16 @@ public class TextIndexConfig extends IndexConfig {
_enablePrefixSuffixMatchingInPhraseQueries =
enablePrefixSuffixMatchingInPhraseQueries;
return this;
}
+
+ public AbstractBuilder withReuseMutableIndex(boolean reuseMutableIndex) {
+ _reuseMutableIndex = reuseMutableIndex;
+ return this;
+ }
+
+ public AbstractBuilder withLuceneNRTCachingDirectoryMaxBufferSizeMB(int
luceneNRTCachingDirectoryMaxBufferSizeMB) {
+ _luceneNRTCachingDirectoryMaxBufferSizeMB =
luceneNRTCachingDirectoryMaxBufferSizeMB;
+ return this;
+ }
}
@Override
@@ -229,16 +262,21 @@ public class TextIndexConfig extends IndexConfig {
}
TextIndexConfig that = (TextIndexConfig) o;
return _enableQueryCache == that._enableQueryCache &&
_useANDForMultiTermQueries == that._useANDForMultiTermQueries
+ && _luceneUseCompoundFile == that._luceneUseCompoundFile
+ && _luceneMaxBufferSizeMB == that._luceneMaxBufferSizeMB
+ && _enablePrefixSuffixMatchingInPhraseQueries ==
that._enablePrefixSuffixMatchingInPhraseQueries
+ && _reuseMutableIndex == that._reuseMutableIndex
+ && _luceneNRTCachingDirectoryMaxBufferSizeMB ==
that._luceneNRTCachingDirectoryMaxBufferSizeMB
&& _fstType == that._fstType && Objects.equals(_rawValueForTextIndex,
that._rawValueForTextIndex)
&& Objects.equals(_stopWordsInclude, that._stopWordsInclude) &&
Objects.equals(_stopWordsExclude,
- that._stopWordsExclude) && _luceneUseCompoundFile ==
that._luceneUseCompoundFile
- && _luceneMaxBufferSizeMB == that._luceneMaxBufferSizeMB &&
_luceneAnalyzerClass == that._luceneAnalyzerClass;
+ that._stopWordsExclude) && Objects.equals(_luceneAnalyzerClass,
that._luceneAnalyzerClass);
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), _fstType, _rawValueForTextIndex,
_enableQueryCache,
_useANDForMultiTermQueries, _stopWordsInclude, _stopWordsExclude,
_luceneUseCompoundFile,
- _luceneMaxBufferSizeMB, _luceneAnalyzerClass);
+ _luceneMaxBufferSizeMB, _luceneAnalyzerClass,
_enablePrefixSuffixMatchingInPhraseQueries, _reuseMutableIndex,
+ _luceneNRTCachingDirectoryMaxBufferSizeMB);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
