Author: frm
Date: Wed Aug 2 13:42:24 2017
New Revision: 1803818
URL: http://svn.apache.org/viewvc?rev=1803818&view=rev
Log:
OAK-6507 - Prevent full head states in use from being removed during cleanup
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Reclaimers.java
(with props)
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ReclaimersTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GCGeneration.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1803818&r1=1803817&r2=1803818&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
Wed Aug 2 13:42:24 2017
@@ -1067,7 +1067,7 @@ public class FileStore extends AbstractF
synchronized void collectBlobReferences(Consumer<String> collector)
throws IOException {
segmentWriter.flush();
tarFiles.collectBlobReferences(collector,
- CompactionResult.newOldReclaimer(getGcGeneration(),
gcOptions.getRetainedGenerations()));
+ Reclaimers.newOldReclaimer(getGcGeneration(),
gcOptions.getRetainedGenerations()));
}
void cancel() {
@@ -1162,7 +1162,7 @@ public class FileStore extends AbstractF
return new CompactionResult(newGeneration) {
@Override
Predicate<GCGeneration> reclaimer() {
- return CompactionResult.newOldReclaimer(newGeneration,
gcOptions.getRetainedGenerations());
+ return Reclaimers.newOldReclaimer(newGeneration,
gcOptions.getRetainedGenerations());
}
@Override
@@ -1188,7 +1188,7 @@ public class FileStore extends AbstractF
return new CompactionResult(currentGeneration) {
@Override
Predicate<GCGeneration> reclaimer() {
- return
CompactionResult.newFailedReclaimer(failedGeneration);
+ return Reclaimers.newExactReclaimer(failedGeneration);
}
@Override
@@ -1209,7 +1209,7 @@ public class FileStore extends AbstractF
return new CompactionResult(currentGeneration) {
@Override
Predicate<GCGeneration> reclaimer() {
- return CompactionResult.newOldReclaimer(currentGeneration,
gcOptions.getRetainedGenerations());
+ return Reclaimers.newOldReclaimer(currentGeneration,
gcOptions.getRetainedGenerations());
}
@Override
@@ -1250,33 +1250,6 @@ public class FileStore extends AbstractF
",reclaim-predicate=" + reclaimer();
}
- private static Predicate<GCGeneration> newFailedReclaimer(@Nonnull
final GCGeneration failedGeneration) {
- return new Predicate<GCGeneration>() {
- @Override
- public boolean apply(GCGeneration generation) {
- return generation.equals(failedGeneration);
- }
- @Override
- public String toString() {
- return "(generation==" + failedGeneration + ")";
- }
- };
- }
-
- private static Predicate<GCGeneration> newOldReclaimer(@Nonnull final
GCGeneration reference, int retainedGenerations) {
- return new Predicate<GCGeneration>() {
- @Override
- public boolean apply(GCGeneration generation) {
- return reference.compareFull(generation) >=
retainedGenerations
- || (reference.compareTail(generation) >=
retainedGenerations && !generation.isTail());
- }
- @Override
- public String toString() {
- // FIXME OAK-3349 align string representation with above
predicate
- return "(" + reference + " - generation >= " +
retainedGenerations + ")";
- }
- };
- }
}
}
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Reclaimers.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Reclaimers.java?rev=1803818&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Reclaimers.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Reclaimers.java
Wed Aug 2 13:42:24 2017
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.base.Predicate;
+import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
+
+class Reclaimers {
+
+ private Reclaimers() {
+ // Prevent instantiation.
+ }
+
+ static Predicate<GCGeneration> newOldReclaimer(@Nonnull final GCGeneration
reference, int retainedGenerations) {
+ return new Predicate<GCGeneration>() {
+
+ @Override
+ public boolean apply(GCGeneration generation) {
+ int deltaFull = reference.compareFull(generation);
+
+ if (deltaFull == 0) {
+ if (generation.getTail() == 0) {
+ return false;
+ }
+
+ int deltaTail = reference.compareTail(generation);
+
+ if (deltaTail == 0) {
+ return false;
+ }
+
+ if (deltaTail >= retainedGenerations) {
+ return !generation.isTail();
+ }
+
+ return false;
+ }
+
+ if (deltaFull >= retainedGenerations) {
+ return true;
+ }
+
+ return generation.getTail() != 0;
+ }
+
+ @Override
+ public String toString() {
+ // FIXME OAK-3349 align string representation with above
predicate
+ return "(" + reference + " - generation >= " +
retainedGenerations + ")";
+ }
+
+ };
+ }
+
+ static Predicate<GCGeneration> newExactReclaimer(@Nonnull final
GCGeneration failedGeneration) {
+ return new Predicate<GCGeneration>() {
+ @Override
+ public boolean apply(GCGeneration generation) {
+ return generation.equals(failedGeneration);
+ }
+ @Override
+ public String toString() {
+ return "(generation==" + failedGeneration + ")";
+ }
+ };
+ }
+
+}
Propchange:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Reclaimers.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GCGeneration.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GCGeneration.java?rev=1803818&r1=1803817&r2=1803818&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GCGeneration.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GCGeneration.java
Wed Aug 2 13:42:24 2017
@@ -74,7 +74,7 @@ public final class GCGeneration {
*/
@Nonnull
public GCGeneration nextFull() {
- return new GCGeneration(full + 1, tail, false);
+ return new GCGeneration(full + 1, 0, false);
}
/**
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java?rev=1803818&r1=1803817&r2=1803818&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
(original)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
Wed Aug 2 13:42:24 2017
@@ -1458,11 +1458,12 @@ public class CompactionAndCleanupIT {
}
@Test
- @Ignore("OAK-6507")
public void latestFullCompactedStateShouldNotBeDeleted() throws Exception {
- try (FileStore fileStore = fileStoreBuilder(getFileStoreFolder())
- .withGCOptions(defaultGCOptions().setEstimationDisabled(true))
- .build()) {
+ SegmentGCOptions gcOptions = defaultGCOptions()
+ .setEstimationDisabled(true)
+ .setRetainedGenerations(2);
+
+ try (FileStore fileStore =
fileStoreBuilder(getFileStoreFolder()).withGCOptions(gcOptions).build()) {
// Create a full, self consistent head state. This state will be
the
// base for the following tail compactions. This increments the
@@ -1480,10 +1481,10 @@ public class CompactionAndCleanupIT {
// Create a tail state on top of the previous tail state. This
// increments the tail generation, but leaves the full generation
- // untouched. The increment in tail generation will make the
cleanup
- // algorithm delete the segments from the full head state. This
- // results in an inconsistent repository and
- // SegmentNotFoundException.
+ // untouched. This brings this generations two generations away
from
+ // the latest full head state. Still, the full head state will not
+ // be deleted because doing so would geenrated an invalid
repository
+ // at risk of SegmentNotFoundException.
fileStore.tailGC();
traverse(fileStore.getHead());
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ReclaimersTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ReclaimersTest.java?rev=1803818&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ReclaimersTest.java
(added)
+++
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ReclaimersTest.java
Wed Aug 2 13:42:24 2017
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file;
+
+import static
org.apache.jackrabbit.oak.segment.file.Reclaimers.newOldReclaimer;
+import static
org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.base.Predicate;
+import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
+import org.junit.Test;
+
+public class ReclaimersTest {
+
+ @Test
+ public void testOldReclaimer() throws Exception {
+ Predicate<GCGeneration> p = newOldReclaimer(newGCGeneration(3, 3,
false), 2);
+
+ // Segments from the same full and tail generation should not be
+ // removed.
+
+ assertFalse(p.apply(newGCGeneration(3, 3, false)));
+ assertFalse(p.apply(newGCGeneration(3, 3, true)));
+
+ // Recent segments, tail or not-tail ones, can't be removed.
+
+ assertFalse(p.apply(newGCGeneration(3, 2, false)));
+ assertFalse(p.apply(newGCGeneration(3, 2, true)));
+
+ // Old segments from the same full generation can be removed as long
+ // as they are not tail segments.
+
+ assertTrue(p.apply(newGCGeneration(3, 1, false)));
+ assertFalse(p.apply(newGCGeneration(3, 1, true)));
+
+ // A full head state from the same full generation can't be removed.
+
+ assertFalse(p.apply(newGCGeneration(3, 0, false)));
+
+ // Some of these generations can be reclaimed, some may not. Since
there
+ // is a new full head state with full generation 3, every tail and
+ // non-tail segment with a tail generation greater than zero can be
+ // removed. The full head state with full generation 2 can't be
removed,
+ // otherwise the condition about the number of retained generations
+ // would be violated.
+
+ assertTrue(p.apply(newGCGeneration(2, 2, false)));
+ assertTrue(p.apply(newGCGeneration(2, 2, true)));
+ assertTrue(p.apply(newGCGeneration(2, 1, false)));
+ assertTrue(p.apply(newGCGeneration(2, 1, true)));
+ assertFalse(p.apply(newGCGeneration(2, 0, false)));
+
+ // These generations can be reclaimed because their full generation is
+ // too old when compared to the number of retained generations.
+
+ assertTrue(p.apply(newGCGeneration(1, 2, false)));
+ assertTrue(p.apply(newGCGeneration(1, 2, true)));
+ assertTrue(p.apply(newGCGeneration(1, 1, false)));
+ assertTrue(p.apply(newGCGeneration(1, 1, true)));
+ assertTrue(p.apply(newGCGeneration(1, 0, false)));
+ }
+
+}
Propchange:
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ReclaimersTest.java
------------------------------------------------------------------------------
svn:eol-style = native
