Author: catholicon
Date: Tue Feb 27 17:42:12 2018
New Revision: 1825475
URL: http://svn.apache.org/viewvc?rev=1825475&view=rev
Log:
OAK-7285: Reindexing using --doc-traversal-mode can OOM while aggregation in
some cases
Incorrect impl in r1825466 (caught thankfully by @chetanm). Fixing that now.
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProviderTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java?rev=1825475&r1=1825474&r2=1825475&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProvider.java
Tue Feb 27 17:42:12 2018
@@ -20,12 +20,12 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
import java.util.Iterator;
+import java.util.Set;
import javax.annotation.Nonnull;
import com.google.common.base.Optional;
import com.google.common.collect.AbstractIterator;
-import com.google.common.collect.Iterables;
import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
import org.apache.jackrabbit.oak.commons.PathUtils;
@@ -46,9 +46,9 @@ import static org.apache.jackrabbit.oak.
class ChildNodeStateProvider {
private final Iterable<NodeStateEntry> entries;
private final String path;
- private final Iterable<String> preferredPathElements;
+ private final Set<String> preferredPathElements;
- public ChildNodeStateProvider(Iterable<NodeStateEntry> entries, String
path, Iterable<String> preferredPathElements) {
+ public ChildNodeStateProvider(Iterable<NodeStateEntry> entries, String
path, Set<String> preferredPathElements) {
this.entries = entries;
this.path = path;
this.preferredPathElements = preferredPathElements;
@@ -60,7 +60,7 @@ class ChildNodeStateProvider {
@Nonnull
public NodeState getChildNode(@Nonnull String name) throws
IllegalArgumentException {
- boolean isPreferred = Iterables.contains(preferredPathElements, name);
+ boolean isPreferred = preferredPathElements.contains(name);
Optional<NodeStateEntry> o = Iterators.tryFind(children(isPreferred),
p -> name.equals(name(p)));
return o.isPresent() ? o.get().getNodeState() : MISSING_NODE;
}
@@ -102,23 +102,23 @@ class ChildNodeStateProvider {
"after main iterator has moved past it", path);
//Prepare an iterator to fetch all child node paths i.e. immediate and
there children
- Iterator<NodeStateEntry> itr = new AbstractIterator<NodeStateEntry>() {
+ return new AbstractIterator<NodeStateEntry>() {
@Override
protected NodeStateEntry computeNext() {
- if (pitr.hasNext() && isAncestor(path, pitr.peek().getPath()))
{
+ while (pitr.hasNext() && isAncestor(path,
pitr.peek().getPath())) {
NodeStateEntry nextEntry = pitr.next();
- String nextEntryName =
PathUtils.getName(nextEntry.getPath());
- if (preferred &&
!Iterables.contains(preferredPathElements, nextEntryName)) {
- return endOfData();
+ String nextEntryPath = nextEntry.getPath();
+ if (isImmediateChild(nextEntryPath)) {
+ String nextEntryName =
PathUtils.getName(nextEntryPath);
+ if (preferred &&
!preferredPathElements.contains(nextEntryName)) {
+ return endOfData();
+ }
+ return nextEntry;
}
- return nextEntry;
}
return endOfData();
}
};
-
- //Filter out non immediate children
- return Iterators.filter(itr, (e) -> isImmediateChild(e.getPath()));
}
private static String name(NodeStateEntry p) {
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java?rev=1825475&r1=1825474&r2=1825475&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileNodeStoreBuilder.java
Tue Feb 27 17:42:12 2018
@@ -22,6 +22,7 @@ package org.apache.jackrabbit.oak.index.
import java.io.File;
import java.io.IOException;
import java.util.Collections;
+import java.util.Set;
import com.google.common.collect.Iterables;
import org.apache.commons.io.FileUtils;
@@ -30,7 +31,7 @@ import org.apache.jackrabbit.oak.spi.blo
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static com.google.common.collect.Iterables.unmodifiableIterable;
+import static java.util.Collections.unmodifiableSet;
public class FlatFileNodeStoreBuilder {
public static final String OAK_INDEXER_USE_ZIP = "oak.indexer.useZip";
@@ -41,7 +42,7 @@ public class FlatFileNodeStoreBuilder {
private final Logger log = LoggerFactory.getLogger(getClass());
private final Iterable<NodeStateEntry> nodeStates;
private final File workDir;
- private Iterable<String> preferredPathElements = Collections.emptySet();
+ private Set<String> preferredPathElements = Collections.emptySet();
private BlobStore blobStore;
private PathElementComparator comparator;
private NodeStateEntryWriter entryWriter;
@@ -60,7 +61,7 @@ public class FlatFileNodeStoreBuilder {
return this;
}
- public FlatFileNodeStoreBuilder withPreferredPathElements(Iterable<String>
preferredPathElements) {
+ public FlatFileNodeStoreBuilder withPreferredPathElements(Set<String>
preferredPathElements) {
this.preferredPathElements = preferredPathElements;
return this;
}
@@ -70,7 +71,7 @@ public class FlatFileNodeStoreBuilder {
comparator = new PathElementComparator(preferredPathElements);
entryWriter = new NodeStateEntryWriter(blobStore);
FlatFileStore store = new FlatFileStore(createdSortedStoreFile(), new
NodeStateEntryReader(blobStore),
- unmodifiableIterable(preferredPathElements), useZip);
+ unmodifiableSet(preferredPathElements), useZip);
if (entryCount > 0) {
store.setEntryCount(entryCount);
}
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java?rev=1825475&r1=1825474&r2=1825475&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStore.java
Tue Feb 27 17:42:12 2018
@@ -23,6 +23,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
+import java.util.Set;
import com.google.common.collect.AbstractIterator;
import com.google.common.io.Closer;
@@ -35,11 +36,11 @@ public class FlatFileStore implements It
private final Closer closer = Closer.create();
private final File storeFile;
private final NodeStateEntryReader entryReader;
- private final Iterable<String> preferredPathElements;
+ private final Set<String> preferredPathElements;
private final boolean compressionEnabled;
private long entryCount = -1;
- public FlatFileStore(File storeFile, NodeStateEntryReader entryReader,
Iterable<String> preferredPathElements, boolean compressionEnabled) {
+ public FlatFileStore(File storeFile, NodeStateEntryReader entryReader,
Set<String> preferredPathElements, boolean compressionEnabled) {
this.storeFile = storeFile;
this.entryReader = entryReader;
this.preferredPathElements = preferredPathElements;
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java?rev=1825475&r1=1825474&r2=1825475&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
Tue Feb 27 17:42:12 2018
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.index.
import java.util.Iterator;
import java.util.ListIterator;
+import java.util.Set;
import com.google.common.collect.AbstractIterator;
import org.apache.commons.collections.list.CursorableLinkedList;
@@ -37,10 +38,10 @@ class FlatFileStoreIterator extends Abst
private final Iterator<NodeStateEntry> baseItr;
private final CursorableLinkedList buffer = new CursorableLinkedList();
private NodeStateEntry current;
- private final Iterable<String> preferredPathElements;
+ private final Set<String> preferredPathElements;
private int maxBufferSize;
- public FlatFileStoreIterator(Iterator<NodeStateEntry> baseItr,
Iterable<String> preferredPathElements) {
+ public FlatFileStoreIterator(Iterator<NodeStateEntry> baseItr, Set<String>
preferredPathElements) {
this.baseItr = baseItr;
this.preferredPathElements = preferredPathElements;
}
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProviderTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProviderTest.java?rev=1825475&r1=1825474&r2=1825475&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProviderTest.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/ChildNodeStateProviderTest.java
Tue Feb 27 17:42:12 2018
@@ -129,6 +129,17 @@ public class ChildNodeStateProviderTest
}
@Test
+ public void allPreferredReadable() {
+ Set<String> preferred = ImmutableSet.of("x", "y");
+ CountingIterable<NodeStateEntry> citr = createList(preferred,
asList("/a", "/a/x", "/a/x/1", "/a/x/2", "/a/x/3",
+ "/a/y"));
+ ChildNodeStateProvider p = new ChildNodeStateProvider(citr, "/a",
preferred);
+
+ assertTrue(p.hasChildNode("x"));
+ assertTrue(p.hasChildNode("y"));
+ }
+
+ @Test
public void childCount() {
Set<String> preferred = ImmutableSet.of("jcr:content", "x");
CountingIterable<NodeStateEntry> citr = createList(preferred,
asList("/a", "/a/jcr:content", "/a/c", "/a/d", "/e", "/e/f"));
