Author: amitj
Date: Mon Jun 25 10:42:16 2018
New Revision: 1834299
URL: http://svn.apache.org/viewvc?rev=1834299&view=rev
Log:
OAK-7584: Add stats for DataStore GC performance
Added stats for candidates found, blobs deleted, size deleted
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/OperationStatsCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1834299&r1=1834298&r2=1834299&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
Mon Jun 25 10:42:16 2018
@@ -493,6 +493,10 @@ public class MarkSweepGarbageCollector i
org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount(deletedSize),
deletedSize);
}
+ statsCollector.updateNumCandidates(count);
+ statsCollector.updateNumDeleted(deleted);
+ statsCollector.updateTotalSizeDeleted(deletedSize);
+
// Remove all the merged marked references
GarbageCollectionType.get(blobStore).removeAllMarkedReferences(blobStore);
LOG.debug("Ending sweep phase of the garbage collector");
@@ -979,15 +983,21 @@ public class MarkSweepGarbageCollector i
}
class GarbageCollectionOperationStats implements OperationsStatsMBean {
- private static final String NAME = "DataStoreGarbageCollection";
- private static final String START = "COUNTER";
- private static final String FINISH_FAILURE = "FAILURE";
- private static final String DURATION = "ACTIVE_TIMER";
- private static final String MARK_DURATION = "MARK_TIMER";
- private static final String SWEEP_DURATION = "SWEEP_TIMER";
+ static final String NAME = "DataStoreGarbageCollection";
+ static final String START = "COUNTER";
+ static final String FINISH_FAILURE = "FAILURE";
+ static final String DURATION = "ACTIVE_TIMER";
+ static final String MARK_DURATION = "MARK_TIMER";
+ static final String SWEEP_DURATION = "SWEEP_TIMER";
+ static final String NUM_BLOBS_DELETED = "NUM_BLOBS_DELETED";
+ static final String TOTAL_SIZE_DELETED = "TOTAL_SIZE_DELETED";
+ static final String NUM_CANDIDATES = "NUM_CANDIDATES";
private CounterStats startCounter;
private CounterStats finishFailureCounter;
+ private CounterStats numDeletedCounter;
+ private CounterStats totalSizeDeletedCounter;
+ private CounterStats numCandidatesCounter;
private TimerStats duration;
private final TimerStats markDuration;
private final TimerStats sweepDuration;
@@ -996,6 +1006,9 @@ public class MarkSweepGarbageCollector i
GarbageCollectionOperationStats(StatisticsProvider sp) {
this.startCounter = sp.getCounterStats(getMetricName(START),
StatsOptions.METRICS_ONLY);
this.finishFailureCounter =
sp.getCounterStats(getMetricName(FINISH_FAILURE), StatsOptions.METRICS_ONLY);
+ this.numDeletedCounter =
sp.getCounterStats(getMetricName(NUM_BLOBS_DELETED), StatsOptions.METRICS_ONLY);
+ this.totalSizeDeletedCounter =
sp.getCounterStats(getMetricName(TOTAL_SIZE_DELETED),
StatsOptions.METRICS_ONLY);
+ this.numCandidatesCounter =
sp.getCounterStats(getMetricName(NUM_CANDIDATES), StatsOptions.METRICS_ONLY);
this.duration = sp.getTimer(getMetricName(DURATION),
StatsOptions.METRICS_ONLY);
this.markDuration = sp.getTimer(getMetricName(MARK_DURATION),
StatsOptions.METRICS_ONLY);
this.sweepDuration = sp.getTimer(getMetricName(SWEEP_DURATION),
StatsOptions.METRICS_ONLY);
@@ -1012,6 +1025,21 @@ public class MarkSweepGarbageCollector i
}
@Override
+ public void updateNumDeleted(long num) {
+ numDeletedCounter.inc(num);
+ }
+
+ @Override
+ public void updateNumCandidates(long num) {
+ numCandidatesCounter.inc(num);
+ }
+
+ @Override
+ public void updateTotalSizeDeleted(long size) {
+ totalSizeDeletedCounter.inc(size);
+ }
+
+ @Override
public void updateDuration(long time, TimeUnit timeUnit) {
duration.update(time, timeUnit);
}
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/OperationStatsCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/OperationStatsCollector.java?rev=1834299&r1=1834298&r2=1834299&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/OperationStatsCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/OperationStatsCollector.java
Mon Jun 25 10:42:16 2018
@@ -32,6 +32,15 @@ public interface OperationStatsCollector
@Override public void finishFailure() {
}
+ @Override public void updateNumDeleted(long num) {
+ }
+
+ @Override public void updateNumCandidates(long num) {
+ }
+
+ @Override public void updateTotalSizeDeleted(long size) {
+ }
+
@Override public void updateDuration(long time, TimeUnit timeUnit) {
}
@@ -53,6 +62,24 @@ public interface OperationStatsCollector
void finishFailure();
/**
+ * Update the number deleted
+ * @param num
+ */
+ void updateNumDeleted(long num);
+
+ /**
+ * Update the number of candidates found
+ * @param num
+ */
+ void updateNumCandidates(long num);
+
+ /**
+ * Update the size deleted
+ * @param size
+ */
+ void updateTotalSizeDeleted(long size);
+
+ /**
* Increment the duration timer
*
* @param time time recorded for the operation
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java?rev=1834299&r1=1834298&r2=1834299&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
(original)
+++
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Mon Jun 25 10:42:16 2018
@@ -34,6 +34,7 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicReference;
@@ -51,6 +52,7 @@ import org.apache.jackrabbit.core.data.D
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.api.jmx.CheckpointMBean;
@@ -70,6 +72,7 @@ import org.apache.jackrabbit.oak.spi.whi
import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
import org.apache.jackrabbit.oak.stats.Clock;
import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
+import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -78,7 +81,19 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.commons.codec.binary.Hex.encodeHexString;
+import static
org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.GarbageCollectionOperationStats
+ .FINISH_FAILURE;
+import static
org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.GarbageCollectionOperationStats.NAME;
+import static
org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.GarbageCollectionOperationStats
+ .NUM_BLOBS_DELETED;
+import static
org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.GarbageCollectionOperationStats
+ .NUM_CANDIDATES;
+import static
org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.GarbageCollectionOperationStats.START;
+import static
org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.GarbageCollectionOperationStats
+ .TOTAL_SIZE_DELETED;
+import static org.apache.jackrabbit.oak.plugins.blob.OperationsStatsMBean.TYPE;
import static
org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType.REPOSITORY;
+import static org.apache.jackrabbit.oak.stats.StatsOptions.METRICS_ONLY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -98,7 +113,9 @@ public class BlobGCTest {
protected BlobReferenceRetriever referenceRetriever;
protected CheckpointMBean checkpointMBean;
-
+ protected ScheduledExecutorService scheduledExecutor;
+ protected ThreadPoolExecutor executor;
+ protected DefaultStatisticsProvider statsProvider;
protected Clock clock;
@Before
@@ -117,6 +134,15 @@ public class BlobGCTest {
};
referenceRetriever = ((MemoryBlobStoreNodeStore)
nodeStore).getBlobReferenceRetriever();
startReferenceTime = ((TimeLapsedBlobStore) blobStore).startTime;
+ scheduledExecutor = Executors.newSingleThreadScheduledExecutor();
+ executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+ statsProvider = new DefaultStatisticsProvider(scheduledExecutor);
+ }
+
+ @After
+ public void after() {
+ new ExecutorCloser(scheduledExecutor).close();
+ new ExecutorCloser(executor).close();
}
protected Clock getClock() {
@@ -134,6 +160,8 @@ public class BlobGCTest {
Set<String> existingAfterGC = gcInternal(0);
assertTrue(Sets.symmetricDifference(state.blobsPresent,
existingAfterGC).isEmpty());
+ assertStats(1, 0, state.blobsAdded.size() - state.blobsPresent.size(),
+ state.blobsAdded.size() - state.blobsPresent.size());
}
@Test
@@ -149,6 +177,7 @@ public class BlobGCTest {
Set<String> existingAfterGC = gcInternal(afterSetupTime -
startReferenceTime + 2);
assertTrue(Sets.symmetricDifference(state.blobsAdded,
existingAfterGC).isEmpty());
+ assertStats(1, 0, 0, state.blobsAdded.size() -
state.blobsPresent.size());
}
@Test
@@ -173,6 +202,9 @@ public class BlobGCTest {
assertEquals(1, customLogs.getLogs().size());
long deletedSize = (state.blobsAdded.size() -
state.blobsPresent.size()) * 100;
assertTrue(customLogs.getLogs().get(0).contains(String.valueOf(deletedSize)));
+ assertStats(1, 0, state.blobsAdded.size() - state.blobsPresent.size(),
+ state.blobsAdded.size() - state.blobsPresent.size());
+ assertEquals(deletedSize, getStatCount(TOTAL_SIZE_DELETED));
customLogs.finished();
assertTrue(Sets.symmetricDifference(state.blobsPresent,
existingAfterGC).isEmpty());
@@ -189,6 +221,7 @@ public class BlobGCTest {
Set<String> existingAfterGC = gcInternal(0, true);
assertTrue(Sets.symmetricDifference(state.blobsAdded,
existingAfterGC).isEmpty());
+ assertStats(1, 0, 0, 0);
}
protected Set<String> gcInternal(long maxBlobGcInSecs) throws Exception {
@@ -196,21 +229,26 @@ public class BlobGCTest {
}
protected Set<String> gcInternal(long maxBlobGcInSecs, boolean markOnly)
throws Exception {
- ThreadPoolExecutor executor = (ThreadPoolExecutor)
Executors.newFixedThreadPool(10);
MarkSweepGarbageCollector gc = initGC(maxBlobGcInSecs, executor);
gc.collectGarbage(markOnly);
assertEquals(0, executor.getTaskCount());
Set<String> existingAfterGC = iterate();
log.info("{} blobs existing after gc : {}", existingAfterGC.size(),
existingAfterGC);
- assertStats(gc.getOperationStats());
return existingAfterGC;
}
- private void assertStats(OperationsStatsMBean operationStats) {
- assertEquals("Start counter mismatch", 1,
operationStats.getStartCount());
- assertEquals("Finish error mismatch", 0,
operationStats.getFailureCount());
+ private void assertStats(int start, int failure, long deleted, long
candidates) {
+ assertEquals("Start counter mismatch", start, getStatCount(START));
+ assertEquals("Finish error mismatch", failure,
getStatCount(FINISH_FAILURE));
+ assertEquals("Num deleted mismatch", deleted,
getStatCount(NUM_BLOBS_DELETED));
+ assertEquals("Num candidates mismatch", candidates,
getStatCount(NUM_CANDIDATES));
+ }
+
+ private long getStatCount(String name) {
+ return statsProvider.getCounterStats(
+ TYPE + "." + NAME + "." + name, METRICS_ONLY).getCount();
}
private MarkSweepGarbageCollector initGC(long blobGcMaxAgeInSecs,
ThreadPoolExecutor executor)
@@ -220,7 +258,6 @@ public class BlobGCTest {
private MarkSweepGarbageCollector initGC(long blobGcMaxAgeInSecs,
ThreadPoolExecutor executor,
String root) throws Exception {
- DefaultStatisticsProvider statsProvider = new
DefaultStatisticsProvider(Executors.newSingleThreadScheduledExecutor());
String repoId = null;
if (SharedDataStoreUtils.isShared(blobStore)) {
repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore);
