This is an automated email from the ASF dual-hosted git repository.
fortino pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new d9ab316b2b OAK-10505 - Make PIPELINED the default indexing strategy.
(#1168)
d9ab316b2b is described below
commit d9ab316b2bca5650b4eb5f50a00ef88c09694f16
Author: Nuno Santos <[email protected]>
AuthorDate: Fri Oct 20 13:39:46 2023 +0200
OAK-10505 - Make PIPELINED the default indexing strategy. (#1168)
* Make PIPELINED the default indexing strategy.
* Remove test method that assumes default strategy is one of the
Traverse*Sort strategy.
---
.../flatfile/FlatFileNodeStoreBuilder.java | 4 +-
.../flatfile/FlatFileNodeStoreBuilderTest.java | 58 ++++++++++++----------
.../document/flatfile/FlatFileStoreTest.java | 5 --
3 files changed, 34 insertions(+), 33 deletions(-)
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
index b6355eddde..6f2c4e6712 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
@@ -131,10 +131,10 @@ public class FlatFileNodeStoreBuilder {
private Predicate<String> pathPredicate = path -> true;
private final Compression algorithm =
IndexStoreUtils.compressionAlgorithm();
- private final boolean useTraverseWithSort =
Boolean.parseBoolean(System.getProperty(OAK_INDEXER_TRAVERSE_WITH_SORT,
"true"));
+ private final boolean useTraverseWithSort =
Boolean.parseBoolean(System.getProperty(OAK_INDEXER_TRAVERSE_WITH_SORT,
"false"));
private final String sortStrategyTypeString =
System.getProperty(OAK_INDEXER_SORT_STRATEGY_TYPE);
private final SortStrategyType sortStrategyType = sortStrategyTypeString
!= null ? SortStrategyType.valueOf(sortStrategyTypeString) :
- (useTraverseWithSort ? SortStrategyType.TRAVERSE_WITH_SORT :
SortStrategyType.STORE_AND_SORT);
+ (useTraverseWithSort ? SortStrategyType.TRAVERSE_WITH_SORT :
SortStrategyType.PIPELINED);
private RevisionVector rootRevision = null;
private DocumentNodeStore nodeStore = null;
private MongoDocumentStore mongoDocumentStore = null;
diff --git
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java
index 0483a6d4e5..0d1465e588 100644
---
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java
+++
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilderTest.java
@@ -22,7 +22,6 @@ import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.IOException;
-import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -36,6 +35,8 @@ import org.apache.jackrabbit.oak.index.IndexerSupport;
import org.apache.jackrabbit.oak.index.indexer.document.CompositeException;
import org.apache.jackrabbit.oak.index.indexer.document.IndexerConfiguration;
import
org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntryTraverserFactory;
+import
org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy;
+import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
import
org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder;
import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
@@ -78,11 +79,15 @@ public class FlatFileNodeStoreBuilderTest {
@Test
public void defaultSortStrategy() throws Exception {
+ MongoDocumentStore mongoDocumentStore = mock(MongoDocumentStore.class);
+ when(mongoDocumentStore.isReadOnly()).thenReturn(true);
FlatFileNodeStoreBuilder builder = new
FlatFileNodeStoreBuilder(folder.getRoot())
- .withLastModifiedBreakPoints(Collections.emptyList())
-
.withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory);
+ .withLastModifiedBreakPoints(List.of())
+
.withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory)
+ .withIndexDefinitions(Set.of())
+ .withMongoDocumentStore(mongoDocumentStore);
SortStrategy sortStrategy =
builder.createSortStrategy(builder.createStoreDir());
- assertTrue(sortStrategy instanceof TraverseWithSortStrategy);
+ assertTrue(sortStrategy instanceof PipelinedStrategy);
}
@Test
@@ -95,12 +100,12 @@ public class FlatFileNodeStoreBuilderTest {
}
@Test
- public void disableTraverseAndSortStrategyUsingSystemProperty() throws
Exception {
- System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "false");
+ public void enableTraverseAndSortStrategyUsingSystemProperty() throws
Exception {
+ System.setProperty(OAK_INDEXER_TRAVERSE_WITH_SORT, "true");
FlatFileNodeStoreBuilder builder = new
FlatFileNodeStoreBuilder(folder.getRoot())
.withNodeStateEntryTraverserFactory(nodeStateEntryTraverserFactory);
SortStrategy sortStrategy =
builder.createSortStrategy(builder.createStoreDir());
- assertTrue(sortStrategy instanceof StoreAndSortStrategy);
+ assertTrue(sortStrategy instanceof TraverseWithSortStrategy);
}
@Test
@@ -128,7 +133,7 @@ public class FlatFileNodeStoreBuilderTest {
System.setProperty(OAK_INDEXER_USE_LZ4, "false");
File newFlatFile = getFile("simple-split.json", Compression.GZIP);
System.setProperty(OAK_INDEXER_SORTED_FILE_PATH,
newFlatFile.getParentFile().getAbsolutePath());
-
+
assertBuild(newFlatFile.getParentFile().getAbsolutePath());
}
@@ -140,7 +145,7 @@ public class FlatFileNodeStoreBuilderTest {
File newFlatFile = getFile("simple-split.json", compression);
System.setProperty(OAK_INDEXER_SORTED_FILE_PATH,
newFlatFile.getParentFile().getAbsolutePath());
-
+
assertBuild(newFlatFile.getParentFile().getAbsolutePath());
}
@@ -150,10 +155,10 @@ public class FlatFileNodeStoreBuilderTest {
File newFlatFile = getFile("complex-split.json", Compression.NONE);
System.setProperty(OAK_INDEXER_SORTED_FILE_PATH,
newFlatFile.getParentFile().getAbsolutePath());
-
+
assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), false);
}
-
+
@Test
public void testBuildListSplit() throws CompositeException, IOException {
System.setProperty(OAK_INDEXER_USE_ZIP, "false");
@@ -161,7 +166,7 @@ public class FlatFileNodeStoreBuilderTest {
File newFlatFile = getFile("complex-split.json", Compression.NONE);
System.setProperty(OAK_INDEXER_SORTED_FILE_PATH,
newFlatFile.getParentFile().getAbsolutePath());
-
+
assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), true);
}
@@ -170,10 +175,10 @@ public class FlatFileNodeStoreBuilderTest {
System.setProperty(OAK_INDEXER_USE_ZIP, "true");
System.setProperty(OAK_INDEXER_USE_LZ4, "false");
System.setProperty(IndexerConfiguration.PROP_OAK_INDEXER_MIN_SPLIT_THRESHOLD,
"0");
-
+
File newFlatFile = getFile("complex-split.json", Compression.GZIP);
System.setProperty(OAK_INDEXER_SORTED_FILE_PATH,
newFlatFile.getParentFile().getAbsolutePath());
-
+
assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), true);
}
@@ -186,22 +191,23 @@ public class FlatFileNodeStoreBuilderTest {
File newFlatFile = getFile("complex-split.json", compression);
System.setProperty(OAK_INDEXER_SORTED_FILE_PATH,
newFlatFile.getParentFile().getAbsolutePath());
-
+
assertBuildList(newFlatFile.getParentFile().getAbsolutePath(), true);
}
public void assertBuild(String dir) throws CompositeException, IOException
{
FlatFileNodeStoreBuilder builder = new
FlatFileNodeStoreBuilder(folder.getRoot()).withNodeStateEntryTraverserFactory(
- nodeStateEntryTraverserFactory);
- FlatFileStore store = builder.build();
- assertEquals(dir, store.getFlatFileStorePath());
+ nodeStateEntryTraverserFactory);
+ try (FlatFileStore store = builder.build()) {
+ assertEquals(dir, store.getFlatFileStorePath());
+ }
}
-
+
private File getFile(String dataFile, Compression compression) throws
IOException {
File flatFile = new
File(getClass().getClassLoader().getResource(dataFile).getFile());
File newFlatFile = new File(folder.getRoot(),
FlatFileStoreUtils.getSortedStoreFileName(compression));
try (BufferedReader reader = FlatFileStoreUtils.createReader(flatFile,
false);
- BufferedWriter writer =
FlatFileStoreUtils.createWriter(newFlatFile, compression)) {
+ BufferedWriter writer =
FlatFileStoreUtils.createWriter(newFlatFile, compression)) {
IOUtils.copy(reader, writer);
}
return newFlatFile;
@@ -209,15 +215,15 @@ public class FlatFileNodeStoreBuilderTest {
public void assertBuildList(String dir, boolean split) throws
CompositeException, IOException {
FlatFileNodeStoreBuilder builder = new
FlatFileNodeStoreBuilder(folder.getRoot()).withNodeStateEntryTraverserFactory(
- nodeStateEntryTraverserFactory);
+ nodeStateEntryTraverserFactory);
IndexHelper indexHelper = mock(IndexHelper.class);
when(indexHelper.getWorkDir()).thenReturn(new File(dir));
IndexerSupport indexerSupport = mock(IndexerSupport.class);
NodeState rootState = mock(NodeState.class);
when(indexerSupport.retrieveNodeStateForCheckpoint()).thenReturn(rootState);
-
+
List<FlatFileStore> storeList = builder.buildList(indexHelper,
indexerSupport, mockIndexDefns());
-
+
if (split) {
assertEquals(new File(dir, "split").getAbsolutePath(),
storeList.get(0).getFlatFileStorePath());
assertTrue(storeList.size() > 1);
@@ -230,7 +236,7 @@ public class FlatFileNodeStoreBuilderTest {
private static Set<IndexDefinition> mockIndexDefns() {
NodeStore store = new MemoryNodeStore();
EditorHook hook = new EditorHook(
- new CompositeEditorProvider(new NamespaceEditorProvider(), new
TypeEditorProvider()));
+ new CompositeEditorProvider(new NamespaceEditorProvider(), new
TypeEditorProvider()));
OakInitializer.initialize(store, new InitialContent(), hook);
Set<IndexDefinition> defns = new HashSet<>();
@@ -243,8 +249,8 @@ public class FlatFileNodeStoreBuilderTest {
NodeTypeInfoProvider mockNodeTypeInfoProvider =
Mockito.mock(NodeTypeInfoProvider.class);
NodeTypeInfo mockNodeTypeInfo = Mockito.mock(NodeTypeInfo.class,
"dam:Asset");
Mockito.when(mockNodeTypeInfo.getNodeTypeName()).thenReturn("dam:Asset");
-
Mockito.when(mockNodeTypeInfoProvider.getNodeTypeInfo("dam:Asset")).thenReturn(mockNodeTypeInfo);
-
+
Mockito.when(mockNodeTypeInfoProvider.getNodeTypeInfo("dam:Asset")).thenReturn(mockNodeTypeInfo);
+
return defns;
}
}
diff --git
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java
index 6ff86050e1..2c54d6953b 100644
---
a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java
+++
b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreTest.java
@@ -140,11 +140,6 @@ public class FlatFileStoreTest {
}
}
- @Test
- public void basicTestDefaultStrategy() throws Exception {
- runBasicTest();
- }
-
@Test
public void parallelDownload() throws Exception {
try {
