Author: thomasm
Date: Thu Jun 25 08:35:56 2020
New Revision: 1879176
URL: http://svn.apache.org/viewvc?rev=1879176&view=rev
Log:
OAK-9121 Oak-run indexing: add a cache to flatfile/PersistedLinkedList
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedList.java
- copied unchanged from r1879175,
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java
- copied, changed from r1879175,
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java
Removed:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedList.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListTest.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java?rev=1879176&r1=1879175&r2=1879176&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileStoreIterator.java
Thu Jun 25 08:35:56 2020
@@ -61,7 +61,7 @@ class FlatFileStoreIterator extends Abst
log.info("Using a key-value store buffer: {}", fileName);
NodeStateEntryReader reader = new NodeStateEntryReader(blobStore);
NodeStateEntryWriter writer = new NodeStateEntryWriter(blobStore);
- this.buffer = new PersistedLinkedList(fileName, writer, reader);
+ this.buffer = new PersistedLinkedList(fileName, writer, reader,
1000);
} else if (memLimitConfig < 0) {
log.info("Setting buffer memory limit unbounded");
this.buffer = new FlatFileBufferLinkedList();
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java?rev=1879176&r1=1879175&r2=1879176&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/NodeStateEntryList.java
Thu Jun 25 08:35:56 2020
@@ -21,7 +21,6 @@ package org.apache.jackrabbit.oak.index.
import java.util.Iterator;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
-import
org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.FlatFileBufferLinkedList.NodeIterator;
import org.jetbrains.annotations.NotNull;
public interface NodeStateEntryList {
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java?rev=1879176&r1=1879175&r2=1879176&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedList.java
Thu Jun 25 08:35:56 2020
@@ -21,6 +21,8 @@ package org.apache.jackrabbit.oak.index.
import java.io.File;
import java.io.IOException;
import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
@@ -44,6 +46,8 @@ public class PersistedLinkedList impleme
private static final String COMPACT_STORE_MILLIS_NAME =
"oak.indexer.linkedList.compactMillis";
+ private final LinkedHashMap<Long, NodeStateEntry> cache;
+
private final NodeStateEntryWriter writer;
private final NodeStateEntryReader reader;
private final String storeFileName;
@@ -53,15 +57,25 @@ public class PersistedLinkedList impleme
private MVStore store;
private MVMap<Long, String> map;
+
private long headIndex;
private long tailIndex;
private long size;
private long lastLog;
private long lastCompact;
+ private long cacheHits, cacheMisses;
- public PersistedLinkedList(String fileName, NodeStateEntryWriter writer,
NodeStateEntryReader reader) {
+ public PersistedLinkedList(String fileName, NodeStateEntryWriter writer,
NodeStateEntryReader reader, int cacheSize) {
LOG.info("Opening store " + fileName);
this.storeFileName = fileName;
+ this.cache =
+ new LinkedHashMap<Long, NodeStateEntry>(cacheSize + 1, .75F,
true) {
+ private static final long serialVersionUID = 1L;
+ @Override
+ public boolean removeEldestEntry(Map.Entry<Long, NodeStateEntry>
eldest) {
+ return size() > cacheSize;
+ }
+ };
File oldFile = new File(fileName);
if (oldFile.exists()) {
LOG.info("Deleting " + fileName);
@@ -86,7 +100,9 @@ public class PersistedLinkedList impleme
public void add(@NotNull NodeStateEntry item) {
Preconditions.checkArgument(item != null, "Can't add null to the
list");
String s = writer.toString(item);
- map.put(tailIndex++, s);
+ long index = tailIndex++;
+ map.put(index, s);
+ cache.put(index, item);
size++;
long sizeBytes = store.getFileStore().size();
long now = System.currentTimeMillis();
@@ -122,17 +138,27 @@ public class PersistedLinkedList impleme
Preconditions.checkState(!isEmpty(), "Cannot remove item from empty
list");
NodeStateEntry ret = get(headIndex);
map.remove(headIndex);
+ cache.remove(headIndex);
headIndex++;
size--;
if (size == 0) {
map.clear();
+ cache.clear();
}
return ret;
}
private NodeStateEntry get(long index) {
- String s = map.get(index);
- return reader.read(s);
+ NodeStateEntry result = cache.get(index);
+ if (result == null) {
+ cacheMisses++;
+ String s = map.get(index);
+ result = reader.read(s);
+ cache.put(index, result);
+ } else {
+ cacheHits++;
+ }
+ return result;
}
@Override
@@ -143,6 +169,7 @@ public class PersistedLinkedList impleme
@Override
public void close() {
store.close();
+ LOG.info("Cache hits {} misses {}", cacheHits, cacheMisses);
}
@Override
Copied:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java
(from r1879175,
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java?p2=jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java&p1=jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java&r1=1879175&r2=1879176&rev=1879176&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/FlatFileBufferLinkedListTest.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/FlatFileBufferLinkedListTest.java
Thu Jun 25 08:35:56 2020
@@ -22,6 +22,7 @@ package org.apache.jackrabbit.oak.index.
import com.google.common.collect.Iterators;
import org.apache.jackrabbit.oak.index.indexer.document.NodeStateEntry;
import
org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.FlatFileBufferLinkedList;
+import
org.apache.jackrabbit.oak.index.indexer.document.flatfile.linkedList.NodeStateEntryList;
import org.junit.Before;
import org.junit.Test;
@@ -187,4 +188,4 @@ public class FlatFileBufferLinkedListTes
private NodeStateEntry testNode(String n) {
return new NodeStateEntry(EMPTY_NODE, n);
}
-}
\ No newline at end of file
+}
Modified:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListTest.java?rev=1879176&r1=1879175&r2=1879176&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListTest.java
(original)
+++
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/linkedList/PersistedLinkedListTest.java
Thu Jun 25 08:35:56 2020
@@ -42,7 +42,7 @@ public class PersistedLinkedListTest ext
BlobStore blobStore = new MemoryBlobStore();
NodeStateEntryReader reader = new NodeStateEntryReader(blobStore);
NodeStateEntryWriter writer = new NodeStateEntryWriter(blobStore);
- list = new PersistedLinkedList(fileName, writer, reader);
+ list = new PersistedLinkedList(fileName, writer, reader, 1);
}
@After
