Author: frm
Date: Mon Nov 6 16:45:25 2017
New Revision: 1814429
URL: http://svn.apache.org/viewvc?rev=1814429&view=rev
Log:
OAK-6883 - Let SizeDeltaGcEstimation take the compaction type into consideration
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimationResult.java
(with props)
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGCEstimationTest.java
(with props)
Removed:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/CompactionEstimatorTest.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimation.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGcEstimation.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1814429&r1=1814428&r2=1814429&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
Mon Nov 6 16:45:25 2017
@@ -368,16 +368,6 @@ public class FileStore extends AbstractF
}
/**
- * Run the compaction gain estimation process.
- * @return
- */
- public GCEstimation estimateCompactionGain() {
- try (ShutDownCloser ignored = shutDown.keepAlive()) {
- return garbageCollector.estimateCompactionGain();
- }
- }
-
- /**
* Copy every referenced record in data (non-bulk) segments. Bulk segments
* are fully kept (they are only removed in cleanup, if there is no
* reference to them).
@@ -624,14 +614,14 @@ public class FileStore extends AbstractF
}
synchronized void runFull() throws IOException {
- run(this::compactFull);
+ run(true, this::compactFull);
}
synchronized void runTail() throws IOException {
- run(this::compactTail);
+ run(false, this::compactTail);
}
- private void run(Supplier<CompactionResult> compact) throws
IOException {
+ private void run(boolean full, Supplier<CompactionResult> compact)
throws IOException {
try {
gcListener.info("TarMK GC #{}: started",
GC_COUNT.incrementAndGet());
@@ -652,9 +642,9 @@ public class FileStore extends AbstractF
gcListener.updateStatus(ESTIMATION.message());
Stopwatch watch = Stopwatch.createStarted();
- GCEstimation estimate = estimateCompactionGain();
- sufficientEstimatedGain = estimate.gcNeeded();
- String gcLog = estimate.gcLog();
+ GCEstimationResult estimation =
estimateCompactionGain(full);
+ sufficientEstimatedGain = estimation.isGcNeeded();
+ String gcLog = estimation.getGcLog();
if (sufficientEstimatedGain) {
gcListener.info(
"TarMK GC #{}: estimation completed in {} ({}
ms). {}",
@@ -696,9 +686,8 @@ public class FileStore extends AbstractF
* the passed {@code stop} signal.
* @return compaction gain estimate
*/
- synchronized GCEstimation estimateCompactionGain() {
- return new SizeDeltaGcEstimation(gcOptions, gcJournal,
- stats.getApproximateSize());
+ GCEstimationResult estimateCompactionGain(boolean full) {
+ return new
SizeDeltaGcEstimation(gcOptions.getGcSizeDeltaEstimation(), gcJournal,
stats.getApproximateSize(), full).estimate();
}
@Nonnull
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimation.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimation.java?rev=1814429&r1=1814428&r2=1814429&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimation.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimation.java
Mon Nov 6 16:45:25 2017
@@ -19,17 +19,8 @@
package org.apache.jackrabbit.oak.segment.file;
-public interface GCEstimation {
+interface GCEstimation {
- /**
- * Determines if the Gc operation needs to run or not
- */
- boolean gcNeeded();
-
- /**
- * User friendly message explaining the value of the
- * {@link GCEstimation#gcNeeded()} flag
- */
- String gcLog();
+ GCEstimationResult estimate();
}
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimationResult.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimationResult.java?rev=1814429&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimationResult.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimationResult.java
Mon Nov 6 16:45:25 2017
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file;
+
+class GCEstimationResult {
+
+ private final boolean gcNeeded;
+
+ private final String gcLog;
+
+ GCEstimationResult(boolean gcNeeded, String gcLog) {
+ this.gcNeeded = gcNeeded;
+ this.gcLog = gcLog;
+ }
+
+ boolean isGcNeeded() {
+ return gcNeeded;
+ }
+
+ String getGcLog() {
+ return gcLog;
+ }
+
+}
Propchange:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimationResult.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGcEstimation.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGcEstimation.java?rev=1814429&r1=1814428&r2=1814429&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGcEstimation.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGcEstimation.java
Mon Nov 6 16:45:25 2017
@@ -23,82 +23,94 @@ import static com.google.common.base.Pre
import static java.lang.String.format;
import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
+import java.util.ArrayList;
+import java.util.List;
+
import javax.annotation.Nonnull;
-import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
+import org.apache.jackrabbit.oak.segment.file.GCJournal.GCJournalEntry;
-public class SizeDeltaGcEstimation implements GCEstimation {
+class SizeDeltaGcEstimation implements GCEstimation {
private final long delta;
private final GCJournal gcJournal;
- private final long totalSize;
-
- private boolean gcNeeded;
-
- private String gcInfo = "unknown";
+ private final long currentSize;
- private boolean finished = false;
+ private final boolean full;
- public SizeDeltaGcEstimation(@Nonnull SegmentGCOptions opts,
- @Nonnull GCJournal gcJournal, long totalSize) {
- this.delta = checkNotNull(opts).getGcSizeDeltaEstimation();
+ SizeDeltaGcEstimation(long delta, @Nonnull GCJournal gcJournal, long
currentSize, boolean full) {
+ this.delta = delta;
this.gcJournal = checkNotNull(gcJournal);
- this.totalSize = totalSize;
+ this.currentSize = currentSize;
+ this.full = full;
}
@Override
- public boolean gcNeeded() {
- if (!finished) {
- run();
+ public GCEstimationResult estimate() {
+ if (delta == 0) {
+ return new GCEstimationResult(true, "Estimation skipped because
the size delta value equals 0");
+ }
+
+ long previousSize = readPreviousSize();
+
+ if (previousSize < 0) {
+ return new GCEstimationResult(true, "Estimation skipped because of
missing gc journal data (expected on first run)");
+ }
+
+ long gain = currentSize - previousSize;
+ boolean gcNeeded = gain > delta;
+ String gcInfo = format(
+ "Segmentstore size has increased since the last %s garbage
collection from %s (%s bytes) to %s (%s bytes), an increase of %s (%s bytes) or
%s%%. ",
+ full ? "full" : "tail",
+ humanReadableByteCount(previousSize), previousSize,
+ humanReadableByteCount(currentSize), currentSize,
+ humanReadableByteCount(gain), gain, 100 * gain / previousSize
+ );
+ if (gcNeeded) {
+ gcInfo = gcInfo + format(
+ "This is greater than sizeDeltaEstimation=%s (%s bytes), so
running garbage collection",
+ humanReadableByteCount(delta), delta
+ );
+ } else {
+ gcInfo = gcInfo + format(
+ "This is less than sizeDeltaEstimation=%s (%s bytes), so
skipping garbage collection",
+ humanReadableByteCount(delta), delta
+ );
}
- return gcNeeded;
+ return new GCEstimationResult(gcNeeded, gcInfo);
}
- @Override
- public String gcLog() {
- if (!finished) {
- run();
+ private long readPreviousSize() {
+ if (full) {
+ return readPreviousFullCleanupSize();
}
- return gcInfo;
+ return readPreviousTailCleanupSize();
}
- private void run() {
- if (finished) {
- return;
+ private long readPreviousFullCleanupSize() {
+ List<GCJournalEntry> entries = new ArrayList<>(gcJournal.readAll());
+
+ if (entries.isEmpty()) {
+ return -1;
}
- if (delta == 0) {
- gcNeeded = true;
- gcInfo = "Estimation skipped because the size delta value equals
0";
- } else if (getPreviousCleanupSize() < 0) {
- gcNeeded = true;
- gcInfo = "Estimation skipped because of missing gc journal data
(expected on first run)";
- } else {
- long lastGc = getPreviousCleanupSize();
- long gain = totalSize - lastGc;
- long gainP = 100 * (totalSize - lastGc) / lastGc;
- gcNeeded = gain > delta;
- gcInfo = format(
- "Segmentstore size has increased since the last garbage
collection from %s (%s bytes) to %s (%s bytes), " +
- "an increase of %s (%s bytes) or %s%%. ",
- humanReadableByteCount(lastGc), lastGc,
- humanReadableByteCount(totalSize), totalSize,
- humanReadableByteCount(gain), gain, gainP);
- if (gcNeeded) {
- gcInfo = gcInfo + format(
- "This is greater than sizeDeltaEstimation=%s (%s
bytes), so running garbage collection",
- humanReadableByteCount(delta), delta);
- } else {
- gcInfo = gcInfo + format(
- "This is less than sizeDeltaEstimation=%s (%s bytes),
so skipping garbage collection",
- humanReadableByteCount(delta), delta);
+
+ entries.sort((a, b) -> {
+ if (a.getGcGeneration().getFullGeneration() >
b.getGcGeneration().getFullGeneration()) {
+ return -1;
}
- }
- finished = true;
+ if (a.getGcGeneration().getFullGeneration() <
b.getGcGeneration().getFullGeneration()) {
+ return 1;
+ }
+ return Integer.compare(a.getGcGeneration().getGeneration(),
b.getGcGeneration().getGeneration());
+ });
+
+ return entries.iterator().next().getRepoSize();
}
- private long getPreviousCleanupSize() {
+ private long readPreviousTailCleanupSize() {
return gcJournal.read().getRepoSize();
}
+
}
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGCEstimationTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGCEstimationTest.java?rev=1814429&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGCEstimationTest.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGCEstimationTest.java
Mon Nov 6 16:45:25 2017
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file;
+
+import static
org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class SizeDeltaGCEstimationTest {
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ private GCJournal journal;
+
+ @Before
+ public void setUpJournal() throws Exception {
+ journal = new GCJournal(folder.getRoot());
+ }
+
+ @Test
+ public void testFullSkippedEstimation() {
+ assertTrue(new SizeDeltaGcEstimation(0, journal, 1000,
true).estimate().isGcNeeded());
+ }
+
+ @Test
+ public void testTailSkippedEstimation() {
+ assertTrue(new SizeDeltaGcEstimation(0, journal, 1000,
false).estimate().isGcNeeded());
+ }
+
+ @Test
+ public void testFullEmptyJournal() {
+ assertTrue(new SizeDeltaGcEstimation(100, journal, 1000,
true).estimate().isGcNeeded());
+ }
+
+ @Test
+ public void testTailEmptyJournal() {
+ assertTrue(new SizeDeltaGcEstimation(100, journal, 1000,
false).estimate().isGcNeeded());
+ }
+
+ @Test
+ public void testTailGCNeeded() throws Exception {
+ journal.persist(100, 1000, newGCGeneration(1, 1, true), 1000, "id");
+ journal.persist(110, 1100, newGCGeneration(2, 1, true), 1000, "id");
+ journal.persist(120, 1200, newGCGeneration(3, 1, true), 1000, "id");
+ assertTrue(new SizeDeltaGcEstimation(50, journal, 1300,
false).estimate().isGcNeeded());
+ }
+
+ @Test
+ public void testTailGCSkipped() throws Exception {
+ journal.persist(100, 1000, newGCGeneration(1, 1, true), 1000, "id");
+ journal.persist(110, 1100, newGCGeneration(2, 1, true), 1000, "id");
+ journal.persist(120, 1200, newGCGeneration(3, 1, true), 1000, "id");
+ assertFalse(new SizeDeltaGcEstimation(200, journal, 1300,
false).estimate().isGcNeeded());
+ }
+
+ @Test
+ public void testFullGCNeeded() throws Exception {
+ journal.persist(100, 1000, newGCGeneration(1, 1, true), 1000, "id");
+ journal.persist(110, 1100, newGCGeneration(2, 1, true), 1000, "id");
+ journal.persist(120, 1200, newGCGeneration(3, 1, true), 1000, "id");
+ journal.persist(130, 1000, newGCGeneration(4, 2, true), 1000, "id");
+ journal.persist(100, 1010, newGCGeneration(5, 2, true), 1000, "id");
+ journal.persist(110, 1020, newGCGeneration(6, 2, true), 1000, "id");
+ assertTrue(new SizeDeltaGcEstimation(100, journal, 1300,
true).estimate().isGcNeeded());
+ }
+
+ @Test
+ public void testFullGCSkipped() throws Exception {
+ journal.persist(100, 1000, newGCGeneration(1, 1, true), 1000, "id");
+ journal.persist(110, 1100, newGCGeneration(2, 1, true), 1000, "id");
+ journal.persist(120, 1200, newGCGeneration(3, 1, true), 1000, "id");
+ journal.persist(130, 1000, newGCGeneration(4, 2, true), 1000, "id");
+ journal.persist(100, 1010, newGCGeneration(5, 2, true), 1000, "id");
+ journal.persist(110, 1020, newGCGeneration(6, 2, true), 1000, "id");
+ assertFalse(new SizeDeltaGcEstimation(100, journal, 1030,
true).estimate().isGcNeeded());
+ }
+
+}
Propchange:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGCEstimationTest.java
------------------------------------------------------------------------------
svn:eol-style = native
