Author: mreutegg
Date: Wed Sep 28 15:27:07 2016
New Revision: 1762684
URL: http://svn.apache.org/viewvc?rev=1762684&view=rev
Log:
OAK-4826: Auto removal of orphaned checkpoints
Merge revisions 1762612,1762632 from trunk
Modified:
jackrabbit/oak/branches/1.4/ (props changed)
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdateTest.java
Propchange: jackrabbit/oak/branches/1.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Sep 28 15:27:07 2016
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740349,1740360,1740625-1740626,1740774,1740837,1740879,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742125,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749424,1749443,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462
,1750465,1750495,1750626,1750809,1750886,1751410,1751445-1751446,1751478,1751753,1751755,1751871,1752198,1752202,1752259,1752273-1752274,1752283,1752292,1752438,1752447-1752448,1752508,1752596,1752616,1752659,1752672,1753262,1753331-1753332,1753335-1753336,1753355,1753444,1754117,1754239,1755157,1755191,1756520,1756580,1757119,1757166,1759433,1760340,1760373,1760387,1760661-1760662,1761412,1761444,1761571,1761762,1761787,1761876,1762453,1762635
+/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735081,1735141,1735267,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738136,1738138,1738207,1738234,1738252,1738775,1738795,1738833,1738950,1738957,1738963,1739712,1739760,1739867,1739894,1739959-1739960,1740114,1740116,1740250,1740333,1740349,1740360,1740625-1740626,1740774,1740837,1740879,1740971,1741016,1741032,1741339,1741343,1742077,1742117,1742125,1742363,1742520,1742888,1742916,1743097,1743172,1743343,1743674,1744265,1744292,1744589,1744670,1744672,1744959,1745038,1745127,1745197,1745336,1745368,1746086,1746117,1746342,1746345,1746408,1746696,1746981,1747198,1747200,1747341-1747342,1747380,1747387,1747406,1747492,1747512,1747654,1748505,1748553,1748722,1748870,1749275,1749350,1749424,1749443,1749464,1749475,1749645,1749662,1749815,1749872,1749875,1749899,1750052,1750076-1750077,1750287,1750457,1750462
,1750465,1750495,1750626,1750809,1750886,1751410,1751445-1751446,1751478,1751753,1751755,1751871,1752198,1752202,1752259,1752273-1752274,1752283,1752292,1752438,1752447-1752448,1752508,1752596,1752616,1752659,1752672,1753262,1753331-1753332,1753335-1753336,1753355,1753444,1754117,1754239,1755157,1755191,1756520,1756580,1757119,1757166,1759433,1760340,1760373,1760387,1760661-1760662,1761412,1761444,1761571,1761762,1761787,1761876,1762453,1762612,1762632,1762635
/jackrabbit/trunk:1345480
Modified:
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java?rev=1762684&r1=1762683&r2=1762684&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
(original)
+++
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdate.java
Wed Sep 28 15:27:07 2016
@@ -30,6 +30,7 @@ import static org.apache.jackrabbit.oak.
import java.io.Closeable;
import java.util.Calendar;
import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
@@ -175,6 +176,28 @@ public class AsyncIndexUpdate implements
private boolean closed;
+ /**
+ * The checkpoint cleanup interval in minutes. Defaults to 5 minutes.
+ * Setting it to a negative value disables automatic cleanup. See OAK-4826.
+ */
+ private final int cleanupIntervalMinutes = getCleanupIntervalMinutes();
+
+ private static int getCleanupIntervalMinutes() {
+ int value = 5;
+ try {
+ value = Integer.parseInt(System.getProperty(
+ "oak.async.checkpointCleanupIntervalMinutes",
String.valueOf(value)));
+ } catch (NumberFormatException e) {
+ // use default
+ }
+ return value;
+ }
+
+ /**
+ * The time in minutes since the epoch when the last checkpoint cleanup
ran.
+ */
+ private long lastCheckpointCleanUpTime;
+
public AsyncIndexUpdate(@Nonnull String name, @Nonnull NodeStore store,
@Nonnull IndexEditorProvider provider, boolean switchOnSync) {
this.name = checkNotNull(name);
@@ -418,6 +441,7 @@ public class AsyncIndexUpdate implements
boolean threadNameChanged = false;
String afterCheckpoint = store.checkpoint(lifetime, ImmutableMap.of(
"creator", AsyncIndexUpdate.class.getSimpleName(),
+ "created", afterTime,
"thread", oldThreadName,
"name", name));
NodeState after = store.retrieve(afterCheckpoint);
@@ -470,6 +494,7 @@ public class AsyncIndexUpdate implements
checkpointToRelease);
}
}
+ maybeCleanUpCheckpoints();
if (updatePostRunStatus) {
postAsyncRunStatsStatus(indexStats);
@@ -477,6 +502,59 @@ public class AsyncIndexUpdate implements
}
}
+ private void maybeCleanUpCheckpoints() {
+ // clean up every five minutes
+ long currentMinutes = TimeUnit.MILLISECONDS.toMinutes(
+ System.currentTimeMillis());
+ if (!indexStats.isFailing()
+ && cleanupIntervalMinutes > -1
+ && lastCheckpointCleanUpTime + cleanupIntervalMinutes <
currentMinutes) {
+ try {
+ cleanUpCheckpoints();
+ } catch (Throwable e) {
+ log.warn("Checkpoint clean up failed", e);
+ }
+ lastCheckpointCleanUpTime = currentMinutes;
+ }
+ }
+
+ void cleanUpCheckpoints() {
+ log.debug("Cleaning up orphaned checkpoints");
+ Set<String> keep = newHashSet();
+ String cp = indexStats.getReferenceCheckpoint();
+ if (cp == null) {
+ log.warn("No reference checkpoint set in index stats");
+ return;
+ }
+ keep.add(cp);
+ keep.addAll(indexStats.tempCps);
+ Map<String, String> info = store.checkpointInfo(cp);
+ String value = info.get("created");
+ if (value != null) {
+ // remove unreferenced AsyncIndexUpdate checkpoints:
+ // - without 'created' info (checkpoint created before OAK-4826)
+ // or
+ // - 'created' value older than the current reference and
+ // not within the lease time frame
+ long current = ISO8601.parse(value).getTimeInMillis();
+ for (String checkpoint : store.checkpoints()) {
+ info = store.checkpointInfo(checkpoint);
+ String creator = info.get("creator");
+ String created = info.get("created");
+ String name = info.get("name");
+ if (!keep.contains(checkpoint)
+ && this.name.equals(name)
+ &&
AsyncIndexUpdate.class.getSimpleName().equals(creator)
+ && (created == null ||
ISO8601.parse(created).getTimeInMillis() + leaseTimeOut < current)) {
+ if (store.release(checkpoint)) {
+ log.info("Removed orphaned checkpoint '{}' {}",
+ checkpoint, info);
+ }
+ }
+ }
+ }
+ }
+
protected AsyncUpdateCallback newAsyncUpdateCallback(NodeStore store,
String name, long
leaseTimeOut, String beforeCheckpoint,
String
afterCheckpoint, AsyncIndexStats indexStats,
@@ -618,6 +696,10 @@ public class AsyncIndexUpdate implements
return this;
}
+ protected long getLeaseTimeOut() {
+ return leaseTimeOut;
+ }
+
protected AsyncIndexUpdate setCloseTimeOut(int timeOutInSec) {
this.softTimeOutSecs = timeOutInSec;
return this;
Modified:
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdateTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdateTest.java?rev=1762684&r1=1762683&r2=1762684&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdateTest.java
(original)
+++
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexUpdateTest.java
Wed Sep 28 15:27:07 2016
@@ -27,12 +27,14 @@ import static org.hamcrest.CoreMatchers.
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.util.ArrayList;
+import java.util.Calendar;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -68,6 +70,8 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.apache.jackrabbit.oak.spi.state.ProxyNodeStore;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.apache.jackrabbit.util.ISO8601;
import org.junit.Test;
import ch.qos.logback.classic.Level;
@@ -673,6 +677,82 @@ public class AsyncIndexUpdateTest {
}
}
+ // OAK-4826
+ @Test
+ public void cpCleanupOrphaned() throws Exception {
+ Clock clock = Clock.SIMPLE;
+ MemoryNodeStore store = new MemoryNodeStore();
+ // prepare index and initial content
+ NodeBuilder builder = store.getRoot().builder();
+ createIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME),
+ "rootIndex", true, false, ImmutableSet.of("foo"), null)
+ .setProperty(ASYNC_PROPERTY_NAME, "async");
+ builder.child("testRoot").setProperty("foo", "abc");
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ assertTrue("Expecting no checkpoints",
+ store.listCheckpoints().size() == 0);
+
+ IndexEditorProvider provider = new PropertyIndexEditorProvider();
+ AsyncIndexUpdate async = new AsyncIndexUpdate("async", store,
provider);
+ async.run();
+ assertTrue("Expecting one checkpoint",
+ store.listCheckpoints().size() == 1);
+ String cp = store.listCheckpoints().iterator().next();
+ Map<String, String> info = store.checkpointInfo(cp);
+
+ builder = store.getRoot().builder();
+ builder.child("testRoot").setProperty("foo", "def");
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+
+ // wait until currentTimeMillis() changes. this ensures
+ // the created value for the checkpoint is different
+ // from the previous checkpoint.
+ clock.waitUntil(clock.getTime() + 1);
+ async.run();
+ assertTrue("Expecting one checkpoint",
+ store.listCheckpoints().size() == 1);
+ cp = store.listCheckpoints().iterator().next();
+
+ // create a new checkpoint with the info from the first checkpoint
+ // this simulates an orphaned checkpoint that should be cleaned up.
+ // the created timestamp is set back in time because cleanup preserves
+ // checkpoints within the lease time frame.
+ Calendar c = Calendar.getInstance();
+ c.setTimeInMillis(clock.getTime() - 2 * async.getLeaseTimeOut());
+ info.put("created", ISO8601.format(c));
+ assertNotNull(store.checkpoint(TimeUnit.HOURS.toMillis(1), info));
+ assertTrue("Expecting two checkpoints",
+ store.listCheckpoints().size() == 2);
+
+ async.cleanUpCheckpoints();
+ assertTrue("Expecting one checkpoint",
+ store.listCheckpoints().size() == 1);
+ assertEquals(cp, store.listCheckpoints().iterator().next());
+ }
+
+ @Test
+ public void disableCheckpointCleanup() throws Exception {
+ String propertyName = "oak.async.checkpointCleanupIntervalMinutes";
+ MemoryNodeStore store = new MemoryNodeStore();
+ IndexEditorProvider provider = new PropertyIndexEditorProvider();
+ try {
+ System.setProperty(propertyName, "-1");
+ final AtomicBoolean cleaned = new AtomicBoolean();
+ AsyncIndexUpdate async = new AsyncIndexUpdate("async", store,
provider) {
+ @Override
+ void cleanUpCheckpoints() {
+ cleaned.set(true);
+ super.cleanUpCheckpoints();
+ }
+ };
+ async.run();
+ assertFalse(cleaned.get());
+ } finally {
+ System.clearProperty(propertyName);
+ }
+ }
+
/**
* OAK-2203 Test reindex behavior on an async index when the index
provider is missing
* for a given type
