Author: catholicon
Date: Thu Mar 1 04:17:43 2018
New Revision: 1825621
URL: http://svn.apache.org/viewvc?rev=1825621&view=rev
Log:
OAK-7290: Reindexing using --doc-traversal-mode should have configurable upper
bound for mem usage
Wire linked list mem limit facility in FlatFileStoreIterator.
Default mem limit is 100MB.
The limit can be configure by setting jvm param 'oak.indexer.memLimitInMB'
Negative value implies unbounded usage (long_max)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java?rev=1825621&r1=1825620&r2=1825621&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
Thu Mar 1 04:17:43 2018
@@ -35,14 +35,25 @@ import static com.google.common.collect.
class FlatFileStoreIterator extends AbstractIterator<NodeStateEntry>
implements Iterator<NodeStateEntry> {
private final Logger log = LoggerFactory.getLogger(getClass());
private final Iterator<NodeStateEntry> baseItr;
- private final FlatFileBufferLinkedList buffer = new
FlatFileBufferLinkedList();
+ private final FlatFileBufferLinkedList buffer;
private NodeStateEntry current;
private final Set<String> preferredPathElements;
private int maxBufferSize;
+ static final String BUFFER_MEM_LIMIT_CONFIG_NAME =
"oak.indexer.memLimitInMB";
+ private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 100;
public FlatFileStoreIterator(Iterator<NodeStateEntry> baseItr, Set<String>
preferredPathElements) {
this.baseItr = baseItr;
this.preferredPathElements = preferredPathElements;
+
+ int memLimitConfig = Integer.getInteger(BUFFER_MEM_LIMIT_CONFIG_NAME,
DEFAULT_BUFFER_MEM_LIMIT_IN_MB);
+ if (memLimitConfig < 0) {
+ log.info("Setting buffer memory limit unbounded", memLimitConfig);
+ this.buffer = new FlatFileBufferLinkedList();
+ } else {
+ log.info("Setting buffer memory limit to {} MBs", memLimitConfig);
+ this.buffer = new FlatFileBufferLinkedList(memLimitConfig * 1024L
* 1024L);
+ }
}
int getBufferSize(){
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java?rev=1825621&r1=1825620&r2=1825621&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIteratorTest.java
Thu Mar 1 04:17:43 2018
@@ -31,10 +31,10 @@ import org.apache.jackrabbit.oak.spi.sta
import org.junit.Test;
import static java.util.Arrays.asList;
+import static
org.apache.jackrabbit.oak.index.indexer.document.flatfile.FlatFileStoreIterator.BUFFER_MEM_LIMIT_CONFIG_NAME;
import static
org.apache.jackrabbit.oak.index.indexer.document.flatfile.TestUtils.createList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class FlatFileStoreIteratorTest {
@@ -158,11 +158,84 @@ public class FlatFileStoreIteratorTest {
NodeStateEntry entry = fitr.next();
NodeState entryNS = entry.getNodeState();
assertEquals("/a", entry.getPath());
- assertEquals("Fetching from iterator doesn't use buffer", 0,
fitr.getBufferSize());
+ assertEquals("Fetching from iterator doesn't use buffer", 0,
fitr.getBufferMemoryUsage());
entryNS.getChildNode("b");
assertEquals(1, fitr.getBufferSize());
assertEquals("Reaching child from node state should estimate 30 for
/a/b",
30, fitr.getBufferMemoryUsage());
}
+
+ @Test
+ public void memUsageConfig() {
+ String configuredValue =
System.clearProperty(BUFFER_MEM_LIMIT_CONFIG_NAME);
+ try {
+ NodeStateEntry root = new
NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/");
+ NodeStateEntry e1Byte = new
NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a/b", 1);
+ NodeStateEntry e1MB = new
NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a", 1 * 1024 * 1024);
+ NodeStateEntry e100MB = new
NodeStateEntry(EmptyNodeState.EMPTY_NODE, "/a", 100 * 1024 * 1024);
+
+ {
+ //default configured limit
+ List<NodeStateEntry> list = Lists.newArrayList(root, e100MB,
e1Byte);
+ FlatFileStoreIterator fitr = new
FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");//default is 100MB,
this should work
+ try {
+ aNS.getChildNode("b");
+ fail("Reading beyond default 100MB must fail");
+ } catch (IllegalStateException ise) {
+ //ignore
+ }
+ }
+
+ {
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "1");
+
+ List<NodeStateEntry> list = Lists.newArrayList(root, e1MB,
e1Byte);
+ FlatFileStoreIterator fitr = new
FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");//configured limit is
10 bytes, this should work
+ try {
+ aNS.getChildNode("b");
+ fail("Reading beyond configured 1MB must fail");
+ } catch (IllegalStateException ise) {
+ //ignore
+ }
+ }
+
+ {
+ // illegal config behaves as default
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "1A");
+
+ List<NodeStateEntry> list = Lists.newArrayList(root, e100MB,
e1Byte);
+ FlatFileStoreIterator fitr = new
FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");//default is 100MB,
this should work
+ try {
+ aNS.getChildNode("b");
+ fail("Reading beyond default 100MB must fail");
+ } catch (IllegalStateException ise) {
+ //ignore
+ }
+ }
+
+ {
+ // negative value for unbounded buffer
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME, "-1");
+
+ List<NodeStateEntry> list = Lists.newArrayList(root, e100MB,
e1Byte);
+ FlatFileStoreIterator fitr = new
FlatFileStoreIterator(list.iterator(), ImmutableSet.of());
+ NodeState rootNS = fitr.next().getNodeState();
+ NodeState aNS = rootNS.getChildNode("a");
+ aNS.getChildNode("b");//configure negative value - mem usage
limit should be unbounded (long_max)
+ }
+ } finally {
+ if (configuredValue == null) {
+ System.clearProperty(BUFFER_MEM_LIMIT_CONFIG_NAME);
+ } else {
+ System.setProperty(BUFFER_MEM_LIMIT_CONFIG_NAME,
configuredValue);
+ }
+ }
+ }
}
\ No newline at end of file
