Author: mduerig
Date: Wed Jul 27 08:52:43 2016
New Revision: 1754228
URL: http://svn.apache.org/viewvc?rev=1754228&view=rev
Log:
OAK-4277: Finalise de-duplication caches
CacheStats MBean for node deduplication cache
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/NodeCache.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentCompactionIT.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/NodeCache.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/NodeCache.java?rev=1754228&r1=1754227&r2=1754228&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/NodeCache.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/NodeCache.java
Wed Jul 27 08:52:43 2016
@@ -32,20 +32,37 @@ import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import com.google.common.base.Supplier;
+import com.google.common.cache.CacheStats;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
// FIXME OAK-4277: Finalise de-duplication caches
-// implement configuration, monitoring and management
+// implement configuration
// add unit tests
// document, nullability
public abstract class NodeCache {
+ private long hitCount;
+ private long missCount;
+ private long loadCount;
+ private long evictionCount;
public abstract void put(String key, RecordId value, int depth);
@CheckForNull
public abstract RecordId get(String key);
+ /**
+ * @return number of mappings
+ */
+ public abstract long size();
+
+ /**
+ * @return access statistics for this cache
+ */
+ public CacheStats getStats() {
+ return new CacheStats(hitCount, missCount, loadCount, 0, 0,
evictionCount);
+ }
+
@Nonnull
public static NodeCache newNodeCache(int capacity, int maxDepth) {
if (capacity <= 0) {
@@ -78,14 +95,22 @@ public abstract class NodeCache {
public synchronized void put(String key, RecordId value, int depth) { }
@Override
- public synchronized RecordId get(String key) { return null; }
+ public synchronized RecordId get(String key) {
+ super.missCount++;
+ return null;
+ }
+
+ @Override
+ public long size() {
+ return 0;
+ }
}
private static class Default extends NodeCache {
private static final Logger LOG =
LoggerFactory.getLogger(Default.class);
private final int capacity;
- private final List<Map<String, RecordId>> nodes;
+ private final List<Map<String, RecordId>> caches;
private int size;
@@ -104,54 +129,61 @@ public abstract class NodeCache {
checkArgument(capacity > 0);
checkArgument(maxDepth > 0);
this.capacity = capacity;
- this.nodes = newArrayList();
+ this.caches = newArrayList();
for (int k = 0; k < maxDepth; k++) {
- nodes.add(new HashMap<String, RecordId>());
+ caches.add(new HashMap<String, RecordId>());
}
}
@Override
public synchronized void put(String key, RecordId value, int depth) {
- // FIXME OAK-4277: Finalise de-duplication caches
- // Validate and optimise the eviction strategy.
- // Nodes with many children should probably get a boost to
- // protecting them from preemptive eviction. Also it might be
- // necessary to implement pinning (e.g. for checkpoints).
while (size >= capacity) {
- int d = nodes.size() - 1;
- int removed = nodes.remove(d).size();
+ int d = caches.size() - 1;
+ int removed = caches.remove(d).size();
size -= removed;
+ super.evictionCount -= removed;
if (removed > 0) {
LOG.info("Evicted cache at depth {} as size {} reached
capacity {}. " +
"New size is {}", d, size + removed, capacity,
size);
}
}
- if (depth < nodes.size()) {
- if (nodes.get(depth).put(key, value) == null) {
+ if (depth < caches.size()) {
+ if (caches.get(depth).put(key, value) == null) {
+ super.loadCount++;
size++;
}
} else {
if (muteDepths.add(depth)) {
LOG.info("Not caching {} -> {} as depth {} reaches or
exceeds the maximum of {}",
- key, value, depth, nodes.size());
+ key, value, depth, caches.size());
}
}
}
@Override
public synchronized RecordId get(String key) {
- for (Map<String, RecordId> map : nodes) {
- if (!map.isEmpty()) {
- RecordId recordId = map.get(key);
+ for (Map<String, RecordId> cache : caches) {
+ if (!cache.isEmpty()) {
+ RecordId recordId = cache.get(key);
if (recordId != null) {
+ super.hitCount++;
return recordId;
}
}
}
+ super.missCount++;
return null;
}
+ @Override
+ public synchronized long size() {
+ long size = 0;
+ for (Map<String, RecordId> cache : caches) {
+ size += cache.size();
+ }
+ return size;
+ }
}
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java?rev=1754228&r1=1754227&r2=1754228&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
Wed Jul 27 08:52:43 2016
@@ -418,8 +418,7 @@ public class SegmentNodeStoreService ext
whiteboard,
CacheStatsMBean.class,
stringDeduplicationCacheStats,CacheStats.TYPE,
- stringDeduplicationCacheStats.getName()
- ));
+ stringDeduplicationCacheStats.getName()));
}
CacheStatsMBean templateDeduplicationCacheStats =
store.getTemplateDeduplicationCacheStats();
@@ -428,8 +427,16 @@ public class SegmentNodeStoreService ext
whiteboard,
CacheStatsMBean.class,
templateDeduplicationCacheStats,CacheStats.TYPE,
- templateDeduplicationCacheStats.getName()
- ));
+ templateDeduplicationCacheStats.getName()));
+ }
+
+ CacheStatsMBean nodeDeduplicationCacheStats =
store.getNodeDeduplicationCacheStats();
+ if (nodeDeduplicationCacheStats != null) {
+ registrations.add(registerMBean(
+ whiteboard,
+ CacheStatsMBean.class,
+ nodeDeduplicationCacheStats,CacheStats.TYPE,
+ nodeDeduplicationCacheStats.getName()));
}
// Listen for Executor services on the whiteboard
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java?rev=1754228&r1=1754227&r2=1754228&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
Wed Jul 27 08:52:43 2016
@@ -149,6 +149,14 @@ public class SegmentWriter {
}
/**
+ * @return Statistics for the node deduplication cache or {@code null} if
not available.
+ */
+ @CheckForNull
+ public CacheStatsMBean getNodeCacheStats() {
+ return cacheManager.getNodeCacheStats();
+ }
+
+ /**
* Write a map record.
* @param base base map relative to which the {@code changes} are
applied ot
* {@code null} for the empty map.
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java?rev=1754228&r1=1754227&r2=1754228&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java
Wed Jul 27 08:52:43 2016
@@ -40,7 +40,7 @@ import com.google.common.cache.CacheStat
import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean;
// FIXME OAK-4277: Finalise de-duplication caches
-// implement configuration, monitoring and management
+// implement configuration
/**
* Instances of this class manage the deduplication caches used
* by the {@link SegmentWriter} to avoid writing multiple copies
@@ -102,6 +102,14 @@ public abstract class WriterCacheManager
}
/**
+ * @return statistics for the node cache or {@code null} if not available.
+ */
+ @CheckForNull
+ public CacheStatsMBean getNodeCacheStats() {
+ return null;
+ }
+
+ /**
* This implementation of {@link WriterCacheManager} returns empty caches
* of size 0.
* @see #INSTANCE
@@ -255,18 +263,19 @@ public abstract class WriterCacheManager
@Override
public CacheStatsMBean getStringCacheStats() {
return new RecordCacheStats("String deduplication cache stats",
- accumulateStats(stringCaches),
accumulateSizes(stringCaches));
+ accumulateRecordCacheStats(stringCaches),
accumulateRecordCacheSizes(stringCaches));
}
@CheckForNull
@Override
public CacheStatsMBean getTemplateCacheStats() {
return new RecordCacheStats("Template deduplication cache stats",
- accumulateStats(templateCaches),
accumulateSizes(templateCaches));
+ accumulateRecordCacheStats(templateCaches),
accumulateRecordCacheSizes(templateCaches));
}
@Nonnull
- private static <T> Supplier<CacheStats> accumulateStats(final
Iterable<RecordCache<T>> caches) {
+ private static <T> Supplier<CacheStats> accumulateRecordCacheStats(
+ final Iterable<RecordCache<T>> caches) {
return new Supplier<CacheStats>() {
@Override
public CacheStats get() {
@@ -280,7 +289,8 @@ public abstract class WriterCacheManager
}
@Nonnull
- public static <T> Supplier<Long> accumulateSizes(final
Generations<RecordCache<T>> caches) {
+ public static <T> Supplier<Long> accumulateRecordCacheSizes(
+ final Iterable<RecordCache<T>> caches) {
return new Supplier<Long>() {
@Override
public Long get() {
@@ -289,6 +299,43 @@ public abstract class WriterCacheManager
size += cache.size();
}
return size;
+ }
+ };
+ }
+
+ @CheckForNull
+ @Override
+ public CacheStatsMBean getNodeCacheStats() {
+ return new RecordCacheStats("Node deduplication cache stats",
+ accumulateNodeCacheStats(nodeCaches),
accumulateNodeCacheSizes(nodeCaches));
+ }
+
+ @Nonnull
+ private static <T> Supplier<CacheStats> accumulateNodeCacheStats(
+ final Iterable<NodeCache> caches) {
+ return new Supplier<CacheStats>() {
+ @Override
+ public CacheStats get() {
+ CacheStats stats = new CacheStats(0, 0, 0, 0, 0, 0);
+ for (NodeCache cache : caches) {
+ stats = stats.plus(cache.getStats());
+ }
+ return stats;
+ }
+ };
+ }
+
+ @Nonnull
+ public static <T> Supplier<Long> accumulateNodeCacheSizes(
+ final Iterable<NodeCache> caches) {
+ return new Supplier<Long>() {
+ @Override
+ public Long get() {
+ long size = 0;
+ for (NodeCache cache : caches) {
+ size += cache.size();
+ }
+ return size;
}
};
}
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1754228&r1=1754227&r2=1754228&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
Wed Jul 27 08:52:43 2016
@@ -424,6 +424,11 @@ public class FileStore implements Segmen
return segmentWriter.getTemplateCacheStats();
}
+ @CheckForNull
+ public CacheStatsMBean getNodeDeduplicationCacheStats() {
+ return segmentWriter.getNodeCacheStats();
+ }
+
public void maybeCompact(boolean cleanup) throws IOException {
gcListener.info("TarMK GC #{}: started", GC_COUNT.incrementAndGet());
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentCompactionIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentCompactionIT.java?rev=1754228&r1=1754227&r2=1754228&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentCompactionIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentCompactionIT.java
Wed Jul 27 08:52:43 2016
@@ -260,6 +260,10 @@ public class SegmentCompactionIT {
assertNotNull(templateDeduplicationCacheStats);
registrations.add(registerMBean(templateDeduplicationCacheStats,
new ObjectName("IT:TYPE=" +
templateDeduplicationCacheStats.getName())));
+ CacheStatsMBean nodeDeduplicationCacheStats =
fileStore.getNodeDeduplicationCacheStats();
+ assertNotNull(nodeDeduplicationCacheStats);
+ registrations.add(registerMBean(nodeDeduplicationCacheStats,
+ new ObjectName("IT:TYPE=" +
nodeDeduplicationCacheStats.getName())));
mBeanRegistration = new CompositeRegistration(registrations);
}
