thomasmueller commented on code in PR #1595:
URL: https://github.com/apache/jackrabbit-oak/pull/1595#discussion_r1694845073
##########
oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java:
##########
@@ -19,34 +19,44 @@
package org.apache.jackrabbit.oak.index.indexer.document.flatfile;
-import static org.apache.jackrabbit.guava.common.collect.Iterators.concat;
-import static
org.apache.jackrabbit.guava.common.collect.Iterators.singletonIterator;
-
-import java.io.Closeable;
-import java.util.Iterator;
-import java.util.Set;
-
+import org.apache.jackrabbit.guava.common.collect.AbstractIterator;
+import org.apache.jackrabbit.oak.commons.IOUtils;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
import
org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.FlatFileBufferLinkedList;
import
org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.NodeStateEntryList;
import
org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.PersistedLinkedList;
+import
org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.PersistedLinkedListV2;
import
org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.ConfigHelper;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.jackrabbit.guava.common.collect.AbstractIterator;
+import java.io.Closeable;
+import java.util.Iterator;
+import java.util.Set;
+
+import static org.apache.jackrabbit.guava.common.collect.Iterators.concat;
+import static
org.apache.jackrabbit.guava.common.collect.Iterators.singletonIterator;
class FlatFileStoreIterator extends AbstractIterator<NodeStateEntry>
implements Iterator<NodeStateEntry>, Closeable {
- private static final Logger log =
LoggerFactory.getLogger(FlatFileStoreIterator.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(FlatFileStoreIterator.class);
static final String BUFFER_MEM_LIMIT_CONFIG_NAME =
"oak.indexer.memLimitInMB";
// by default, use the PersistedLinkedList
private static final int DEFAULT_BUFFER_MEM_LIMIT_IN_MB = 0;
- static final String PERSISTED_LINKED_LIST_CACHE_SIZE =
"oak.indexer.persistedLinkedList.cacheSize";
- static final int DEFAULT_PERSISTED_LINKED_LIST_CACHE_SIZE = 1000;
+ public static final String PERSISTED_LINKED_LIST_CACHE_SIZE =
"oak.indexer.persistedLinkedList.cacheSize";
+ public static final int DEFAULT_PERSISTED_LINKED_LIST_CACHE_SIZE = 1000;
+
+ public static final String PERSISTED_LINKED_LIST_V2_CACHE_SIZE =
"oak.indexer.persistedLinkedListV2.cacheSize";
+ public static final int DEFAULT_PERSISTED_LINKED_LIST_V2_CACHE_SIZE =
10000;
+
+ public static final String PERSISTED_LINKED_LIST_V2_MEMORY_CACHE_SIZE_MB =
"oak.indexer.persistedLinkedListV2.cacheMaxSizeMB";
+ public static final int
DEFAULT_PERSISTED_LINKED_LIST_V2_MEMORY_CACHE_SIZE_MB = 8;
+
+ public static final String PERSISTED_LINKED_LIST_USE_V2 =
"oak.indexer.persistedLinkedList.useV2";
Review Comment:
> This is to account for the case where entries are all very small.
What I usually do is add some offset to the memory calculation... basically
"+ 100" or so. (in the past I also verified that the memory calculation is
somewhat correct, by have a small program that runs out-of-memory... but I
understand this can't be a unit test, and we want to support multiple JVM
versions without having to test each time... so the memory estimation of course
won't ever be accurate... but it doesn't have to be)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
