This is an automated email from the ASF dual-hosted git repository.
stefanegli pushed a commit to branch DetailedGC/OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/DetailedGC/OAK-10199 by this
push:
new c710cd698b OAK-10743 : gap orphan mode introduced - gc modes added,
refined and … (#1404)
c710cd698b is described below
commit c710cd698b10bc50193a18d6feca078193d5a170
Author: stefan-egli <[email protected]>
AuthorDate: Thu Apr 4 19:56:58 2024 +0200
OAK-10743 : gap orphan mode introduced - gc modes added, refined and …
(#1404)
* OAK-10743 : gap orphan mode introduced - gc modes added, refined and
renamed
* OAK-10743 : fix compilation error - testRecentRevisionsArePreserved fails
now however
---
.../document/NodeDocumentRevisionCleaner.java | 33 +-
.../plugins/document/VersionGarbageCollector.java | 146 ++++-
.../oak/plugins/document/BranchCommitGCTest.java | 259 ++++++---
.../oak/plugins/document/DetailGCHelper.java | 9 +-
.../document/VersionGarbageCollectorIT.java | 603 ++++++++++++++++-----
5 files changed, 804 insertions(+), 246 deletions(-)
diff --git
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java
index eb6d604d41..59d32fbe4e 100644
---
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java
+++
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java
@@ -18,7 +18,11 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Key;
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation;
+import org.apache.jackrabbit.oak.plugins.document.UpdateOp.Operation.Type;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+import org.jetbrains.annotations.NotNull;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
@@ -26,6 +30,7 @@ import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
+import java.util.Map.Entry;
/**
* This is a prototype class of a very fine-grained revision cleaner that
cleans even revisions
@@ -40,6 +45,7 @@ public class NodeDocumentRevisionCleaner {
private final NodeDocument workingDocument;
private final RevisionPropertiesClassifier revisionClassifier;
private final RevisionCleanerUtility revisionCleaner;
+ private long toModifiedMs;
/**
* Constructor for NodeDocumentRevisionCleaner.
@@ -47,8 +53,18 @@ public class NodeDocumentRevisionCleaner {
* @param workingDocument The document to clean up.
*/
public NodeDocumentRevisionCleaner(DocumentNodeStore documentNodeStore,
NodeDocument workingDocument) {
+ this(documentNodeStore, workingDocument, Instant.now().minus(24,
ChronoUnit.HOURS).toEpochMilli());
+ }
+
+ /**
+ * Constructor for NodeDocumentRevisionCleaner.
+ * @param documentNodeStore The DocumentNodeStore instance.
+ * @param workingDocument The document to clean up.
+ */
+ public NodeDocumentRevisionCleaner(DocumentNodeStore documentNodeStore,
NodeDocument workingDocument, long toModifiedMs) {
this.workingDocument = workingDocument;
this.documentNodeStore = documentNodeStore;
+ this.toModifiedMs = toModifiedMs;
revisionClassifier = new RevisionPropertiesClassifier(workingDocument);
revisionCleaner = new RevisionCleanerUtility(revisionClassifier);
@@ -68,11 +84,22 @@ public class NodeDocumentRevisionCleaner {
for (Revision revision : entry.getValue()) {
TreeSet<String> properties =
revisionClassifier.getPropertiesModifiedByRevision().get(revision);
if (properties != null) {
- for (String property : properties) {
+ outer:for (String property : properties) {
+ Map<Key, Operation> c = op.getChanges();
+ for (Entry<Key, Operation> e : c.entrySet()) {
+ if (e.getKey().equals(new Key(property, null)) &&
e.getValue().type == Type.REMOVE) {
+ continue outer;
+ }
+ }
op.removeMapEntry(property, revision);
}
}
- op.removeMapEntry("_revisions", revision);
+ RevisionVector sweepRevisions =
documentNodeStore.getSweepRevisions();
+ boolean newerThanSweep = sweepRevisions == null ? false :
sweepRevisions.isRevisionNewer(revision);
+ boolean isBC =
workingDocument.getLocalBranchCommits().contains(revision);
+ if (!newerThanSweep && !isBC) {
+ op.removeMapEntry("_revisions", revision);
+ }
}
}
}
@@ -176,7 +203,7 @@ public class NodeDocumentRevisionCleaner {
}
private void preserveRevisionsNewerThanThreshold(long amount,
ChronoUnit unit) {
- long thresholdToPreserve = Instant.now().minus(amount,
unit).toEpochMilli();
+ long thresholdToPreserve =
toModifiedMs;//Instant.now().minus(amount, unit).toEpochMilli();
for (TreeSet<Revision> revisionSet :
candidateRevisionsToClean.values()) {
for (Revision revision : revisionSet) {
if (revision.getTimestamp() > thresholdToPreserve) {
diff --git
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index 8aece8136d..18db77b122 100644
---
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -38,6 +38,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
+import java.util.stream.Stream;
import org.apache.jackrabbit.guava.common.base.Function;
import org.apache.jackrabbit.guava.common.base.Joiner;
@@ -150,17 +151,50 @@ public class VersionGarbageCollector {
*/
static final String
SETTINGS_COLLECTION_DETAILED_GC_DRY_RUN_DOCUMENT_ID_PROP = "detailedGCDryRunId";
- enum RDGCType {
- NO_OLD_PROP_REV_GC,
- KEEP_ONE_FULL_MODE,
- KEEP_ONE_CLEANUP_USER_PROPERTIES_ONLY_MODE,
- OLDER_THAN_24H_AND_BETWEEN_CHECKPOINTS_MODE
+ /**
+ * During hardening of DetailedGC one can choose level type of garbage
should be cleaned up.
+ * Ultimately the goal is to clean up all possible garbage. After
hardening these modes
+ * might no longer be supported.
+ */
+ enum DetailedGCMode {
+ /** no detailed GC is done at all */
+ NONE,
+ /** GC only orphaned nodes with gaps in ancestor docs */
+ GAP_ORPHANS,
+ /** GC orphaned nodes with gaps in ancestor docs, plus empty
properties */
+ GAP_ORPHANS_EMPTYPROPS,
+ /** GC any kind of orphaned nodes, plus empty properties */
+ ALL_ORPHANS_EMPTYPROPS,
+ /**
+ * GC any kind of orphaned nodes, empty properties plus keep 1 (== keep
+ * traversed) revision, applied to user properties only
+ */
+ ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS,
+ /**
+ * GC any kind of orphaned nodes, empty properties plus keep 1 (== keep
+ * traversed) revision, applied to all properties
+ */
+ ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS,
+ /**
+ * GC any kind of orphaned nodes, empty properties plus cleanup
unmerged BCs
+ */
+ ORPHANS_EMPTYPROPS_UNMERGED_BC,
+ /**
+ * GC any kind of orphaned nodes, empty properties plus cleanup
revisions, also
+ * between checkpoints
+ */
+ ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC,
+ /**
+ * GC any kind of orphaned nodes, empty properties, cleanup revisions,
also
+ * between checkpoints, plus cleanup unmerged BCs
+ */
+ ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC
}
- private static RDGCType revisionDetailedGcType =
RDGCType.NO_OLD_PROP_REV_GC;
+ private static DetailedGCMode detailedGcMode =
DetailedGCMode.GAP_ORPHANS_EMPTYPROPS;
- static RDGCType getRevisionDetailedGcType() {
- return revisionDetailedGcType;
+ static DetailedGCMode getDetailedGcMode() {
+ return detailedGcMode;
}
private final DocumentNodeStore nodeStore;
@@ -187,7 +221,7 @@ public class VersionGarbageCollector {
this.isDetailedGCDryRun = isDetailedGCDryRun;
this.embeddedVerification = embeddedVerification;
this.options = new VersionGCOptions();
- log.info("<init> VersionGarbageCollector created with
revisionDetailedGcType = {}", revisionDetailedGcType);
+ log.info("<init> VersionGarbageCollector created with
revisionDetailedGcType = {}", detailedGcMode);
}
void setStatisticsProvider(StatisticsProvider provider) {
@@ -1003,6 +1037,10 @@ public class VersionGarbageCollector {
}
public void collectGarbage(final NodeDocument doc, final GCPhases
phases) {
+ if (detailedGcMode == DetailedGCMode.NONE) {
+ // TODO we should avoid getting here in the first case then
probably
+ return;
+ }
detailedGCStats.documentRead();
monitor.info("Collecting Detailed Garbage for doc [{}]",
doc.getId());
@@ -1017,25 +1055,51 @@ public class VersionGarbageCollector {
if (!isDeletedOrOrphanedNode(traversedState, phases, doc)) {
// here the node is not orphaned which means that we can reach
the node from root
- collectDeletedProperties(doc, phases, op, traversedState);
- switch(revisionDetailedGcType) {
- case NO_OLD_PROP_REV_GC : {
+ switch(detailedGcMode) {
+ case NONE : {
+ // shouldn't be reached
+ return;
+ }
+ case GAP_ORPHANS : {
+ // this mode does neither unusedproprev, nor unmergedBC
+ break;
+ }
+ case GAP_ORPHANS_EMPTYPROPS : {
+ collectDeletedProperties(doc, phases, op,
traversedState);
// this mode does neither unusedproprev, nor unmergedBC
break;
}
- case KEEP_ONE_FULL_MODE : {
+ case ALL_ORPHANS_EMPTYPROPS : {
+ collectDeletedProperties(doc, phases, op,
traversedState);
+ // this mode does neither unusedproprev, nor unmergedBC
+ break;
+ }
+ case ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS : {
+ collectDeletedProperties(doc, phases, op,
traversedState);
collectUnusedPropertyRevisions(doc, phases, op,
(DocumentNodeState) traversedState, false);
combineInternalPropRemovals(doc, op);
break;
}
- case KEEP_ONE_CLEANUP_USER_PROPERTIES_ONLY_MODE : {
+ case ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS : {
+ collectDeletedProperties(doc, phases, op,
traversedState);
collectUnusedPropertyRevisions(doc, phases, op,
(DocumentNodeState) traversedState, true);
combineInternalPropRemovals(doc, op);
break;
}
- case OLDER_THAN_24H_AND_BETWEEN_CHECKPOINTS_MODE : {
+ case ORPHANS_EMPTYPROPS_UNMERGED_BC : {
+ collectDeletedProperties(doc, phases, op,
traversedState);
+ collectUnmergedBranchCommits(doc, phases, op,
toModifiedMs);
+ break;
+ }
+ case
ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC : {
+ collectDeletedProperties(doc, phases, op,
traversedState);
collectUnmergedBranchCommits(doc, phases, op,
toModifiedMs);
- collectRevisionsOlderThan24hAndBetweenCheckpoints(doc,
phases, op);
+ collectRevisionsOlderThan24hAndBetweenCheckpoints(doc,
toModifiedMs, phases, op);
+ break;
+ }
+ case ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC
: {
+ collectDeletedProperties(doc, phases, op,
traversedState);
+ collectRevisionsOlderThan24hAndBetweenCheckpoints(doc,
toModifiedMs, phases, op);
break;
}
}
@@ -1122,6 +1186,29 @@ public class VersionGarbageCollector {
phases.stop(GCPhase.DETAILED_GC_COLLECT_ORPHAN_NODES);
return false;
}
+ if (detailedGcMode == DetailedGCMode.GAP_ORPHANS
+ || detailedGcMode ==
DetailedGCMode.GAP_ORPHANS_EMPTYPROPS) {
+ // check the ancestor docs for gaps
+ boolean hasGaps = false;
+ boolean parentDocExists = true;
+ Path path = Path.ROOT;
+ for (String name : doc.getPath().elements()) {
+ if (!parentDocExists) {
+ hasGaps = true;
+ break;
+ }
+ path = new Path(path, name);
+ final NodeDocument d =
nodeStore.getDocumentStore().find(NODES,
+ Utils.getIdFromPath(path));
+ parentDocExists = d != null;
+ }
+ if (!hasGaps) {
+ // then it is not a gap orphan
+ // nothing to do here then
+ phases.stop(GCPhase.DETAILED_GC_COLLECT_ORPHAN_NODES);
+ return false;
+ }
+ }
// if this is an orphaned node, all that is needed is its removal
garbageDocsCount++;
@@ -1221,7 +1308,13 @@ public class VersionGarbageCollector {
.filter(e -> filterEmptyProps(doc, e.getKey(),
e.getValue()))
.mapToInt(e -> {
final String prop = e.getKey();
+ final int origCount = updateOp.getChanges().size();
updateOp.getChanges().entrySet().removeIf(opEntry
-> Objects.equals(prop, opEntry.getKey().getName()));
+ final int diff = origCount -
updateOp.getChanges().size();
+ //TODO: beautify once this is in DetailedGC branch
+ if (diff > 0) {
+
deletedInternalPropRevsCountMap.merge(doc.getId(), -diff, Integer::sum);
+ }
updateOp.remove(prop);
return 1;})
.sum();
@@ -1385,15 +1478,32 @@ public class VersionGarbageCollector {
}
- private void collectRevisionsOlderThan24hAndBetweenCheckpoints(final
NodeDocument doc,
+ private void collectRevisionsOlderThan24hAndBetweenCheckpoints(final
NodeDocument doc, final long toModifiedMs,
final GCPhases
phases, final UpdateOp updateOp) {
if (phases.start(GCPhase.DETAILED_GC_COLLECT_OLD_REVS)){
- NodeDocumentRevisionCleaner cleaner = new
NodeDocumentRevisionCleaner(nodeStore, doc);
+ NodeDocumentRevisionCleaner cleaner = new
NodeDocumentRevisionCleaner(nodeStore, doc, toModifiedMs);
+ final int beforeRevs = countRevs(updateOp, false);
+ final int beforeIntRevs = countRevs(updateOp, true);
cleaner.collectOldRevisions(updateOp);
+ final int revsDiff = countRevs(updateOp, false) - beforeRevs;
+ if (revsDiff > 0) {
+ deletedPropRevsCountMap.merge(doc.getId(), revsDiff,
Integer::sum);
+ }
+ final int intRevsDiff = countRevs(updateOp, true) -
beforeIntRevs;
+ if (intRevsDiff > 0) {
+ deletedInternalPropRevsCountMap.merge(doc.getId(),
intRevsDiff, Integer::sum);
+ }
phases.stop(GCPhase.DETAILED_GC_COLLECT_OLD_REVS);
}
}
+ private int countRevs(UpdateOp updateOp, boolean internalProps) {
+ return stream(updateOp.getChanges().entrySet().spliterator(),
false)
+ .filter(e -> (e.getValue().type == Type.REMOVE_MAP_ENTRY))
+ .filter(e -> (e.getKey().getName().startsWith("_") ==
internalProps))
+ .mapToInt(x -> 1).sum();
+ }
+
/**
* Remove all property revisions in the local document that are no
longer used.
* This includes bundled properties. It also includes related entries
that
diff --git
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java
index 36d58d589e..5ae5bf83c1 100644
---
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java
+++
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/BranchCommitGCTest.java
@@ -16,8 +16,40 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
+import static java.util.concurrent.TimeUnit.HOURS;
+import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
+import static org.apache.commons.lang3.reflect.FieldUtils.writeStaticField;
+import static
org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.assertBranchRevisionRemovedFromAllDocuments;
+import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.build;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.allOrphProp;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.assertStatsCountsEqual;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.assertStatsCountsZero;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.betweenChkp;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.btwnChkpUBC;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.gapOrphOnly;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.gapOrphProp;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.isModeOneOf;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.keepOneFull;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.keepOneUser;
+import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.unmergedBcs;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.SortedMap;
+import java.util.function.Consumer;
+
import org.apache.jackrabbit.oak.plugins.document.DocumentMK.Builder;
-import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.RDGCType;
+import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.DetailedGCMode;
+import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.GCCounts;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
@@ -31,32 +63,6 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.SortedMap;
-import java.util.function.Consumer;
-
-import static java.util.concurrent.TimeUnit.HOURS;
-import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
-import static org.apache.commons.lang3.reflect.FieldUtils.writeStaticField;
-import static
org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.assertBranchRevisionRemovedFromAllDocuments;
-import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.build;
-import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.noOldPropGc;
-import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.keepOneFull;
-import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.keepOneUser;
-import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.betweenChkp;
-import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.assertStatsCountsEqual;
-import static
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollectorIT.assertStatsCountsZero;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
@RunWith(Parameterized.class)
public class BranchCommitGCTest {
@@ -72,7 +78,7 @@ public class BranchCommitGCTest {
java.util.Collection<Object[]> params = new LinkedList<>();
for (Object[] fixture : AbstractDocumentStoreTest.fixtures()) {
DocumentStoreFixture f = (DocumentStoreFixture)fixture[0];
- for (RDGCType gcType : RDGCType.values()) {
+ for (DetailedGCMode gcType : DetailedGCMode.values()) {
params.add(new Object[] {f, gcType});
}
}
@@ -83,9 +89,9 @@ public class BranchCommitGCTest {
public DocumentStoreFixture fixture;
@Parameter(1)
- public RDGCType gcType;
+ public DetailedGCMode detailedGcMode;
- private RDGCType originalGcType;
+ private DetailedGCMode originalDetailedGcMode;
@Before
public void setUp() throws Exception {
@@ -96,13 +102,13 @@ public class BranchCommitGCTest {
.setLeaseCheckMode(LeaseCheckMode.DISABLED).setDetailedGCEnabled(true)
.setAsyncDelay(0).getNodeStore();
gc = store.getVersionGarbageCollector();
- originalGcType = VersionGarbageCollector.getRevisionDetailedGcType();
- writeStaticField(VersionGarbageCollector.class,
"revisionDetailedGcType", gcType, true);
+ originalDetailedGcMode = VersionGarbageCollector.getDetailedGcMode();
+ writeStaticField(VersionGarbageCollector.class, "detailedGcMode",
detailedGcMode, true);
}
@After
public void tearDown() throws Exception {
- writeStaticField(VersionGarbageCollector.class,
"revisionDetailedGcType", originalGcType, true);
+ writeStaticField(VersionGarbageCollector.class, "detailedGcMode",
originalDetailedGcMode, true);
assertNoEmptyProperties();
if (store != null) {
store.dispose();
@@ -129,18 +135,24 @@ public class BranchCommitGCTest {
VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(2, 0, 0, 0, 0, 0, 2),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(2, 0, 0, 0, 0, 0, 2),
keepOneFull(2, 0, 1, 0, 1, 0, 3),
keepOneUser(2, 0, 0, 0, 0, 0, 2),
- betweenChkp(2, 0, 1, 0, 2, 1, 3));
+ betweenChkp(2, 0, 0, 0, 0, 0, 2),
+ unmergedBcs(2, 0, 1, 0, 1, 1, 3),
+ btwnChkpUBC(2, 0, 1, 0, 1, 1, 3));
// now do another gc - should not have anything left to clean up though
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
stats = gc.gc(1, HOURS);
assertStatsCountsZero(stats);
- assertNotExists("1:/a");
- assertNotExists("1:/b");
- assertBranchRevisionRemovedFromAllDocuments(store, br);
+ if (!isModeOneOf(DetailedGCMode.NONE, DetailedGCMode.GAP_ORPHANS,
DetailedGCMode.GAP_ORPHANS_EMPTYPROPS)) {
+ assertNotExists("1:/a");
+ assertNotExists("1:/b");
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
}
@Test
@@ -182,17 +194,29 @@ public class BranchCommitGCTest {
// expects a collision to have happened - which was cleaned up -
hence a _bc (but not the _revision I guess)
// the collisions cleaned up - with the 1 collision and a _bc
assertStatsCountsEqual(stats,
- noOldPropGc(2, 0, 0, 0, 0, 0, 0),
+ new GCCounts(DetailedGCMode.NONE,
+ 2, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(2, 0, 0, 0, 0, 0, 0),
+ gapOrphProp(2, 0, 0, 0, 0, 0, 0),
+ allOrphProp(2, 0, 0, 0, 0, 0, 0),
keepOneFull(2, 0, 1, 0, 2, 0, 1),
keepOneUser(2, 0, 0, 0, 0, 0, 0),
- betweenChkp(2, 0, 1, 0, 3, 1, 1));
+ betweenChkp(2, 0, 0, 0, 0, 0, 0),
+ unmergedBcs(2, 0, 1, 0, 2, 1, 1),
+ btwnChkpUBC(2, 0, 1, 0, 2, 1, 1));
} else {
// in this case classic collision cleanup already took care of
everything, nothing left
assertStatsCountsEqual(stats,
- noOldPropGc(2, 0, 0, 0, 0, 0, 0),
+ new GCCounts(DetailedGCMode.NONE,
+ 2, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(2, 0, 0, 0, 0, 0, 0),
+ gapOrphProp(2, 0, 0, 0, 0, 0, 0),
+ allOrphProp(2, 0, 0, 0, 0, 0, 0),
keepOneFull(2, 1, 0, 0, 0, 0, 0),
keepOneUser(2, 1, 0, 0, 0, 0, 0),
- betweenChkp(2, 1, 0, 0, 0, 0, 0));
+ betweenChkp(2, 0, 0, 0, 0, 0, 0),
+ unmergedBcs(2, 0, 0, 0, 0, 0, 0),
+ btwnChkpUBC(2, 0, 0, 0, 0, 0, 0));
}
assertNotExists("1:/a");
@@ -247,14 +271,20 @@ public class BranchCommitGCTest {
// 6 deleted props: 0:/[_collisions], 1:/foo[p, a],
1:/bar[_bc,prop,_revisions]
assertStatsCountsEqual(stats,
- noOldPropGc(0, 3, 0, 0, 0, 0, 2),
+ gapOrphOnly(),
+ gapOrphProp(0, 3, 0, 0, 0, 0, 2),
+ allOrphProp(0, 3, 0, 0, 0, 0, 2),
keepOneFull(0, 3, 2, 1,17, 0, 3),
keepOneUser(0, 3, 0, 1, 0, 0, 2),
- betweenChkp(0, 3, 2, 1,18, 4, 3));
- assertBranchRevisionRemovedFromAllDocuments(store, br1);
- assertBranchRevisionRemovedFromAllDocuments(store, br2);
- assertBranchRevisionRemovedFromAllDocuments(store, br3);
- assertBranchRevisionRemovedFromAllDocuments(store, br4);
+ betweenChkp(0, 3, 0, 0, 1, 0, 3),
+ unmergedBcs(0, 3, 2, 1,15, 4, 3),
+ btwnChkpUBC(0, 3, 2, 1,16, 4, 3));
+ if (!isModeOneOf(DetailedGCMode.NONE, DetailedGCMode.GAP_ORPHANS)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store, br2);
+ assertBranchRevisionRemovedFromAllDocuments(store, br3);
+ assertBranchRevisionRemovedFromAllDocuments(store, br4);
+ }
}
@Test
@@ -280,20 +310,28 @@ public class BranchCommitGCTest {
VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(2, 0, 0, 0, 0, 0, 2),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(2, 0, 0, 0, 0, 0, 2),
keepOneFull(2, 0, 2, 0, 2, 0, 3),
keepOneUser(2, 0, 0, 0, 0, 0, 2),
- betweenChkp(2, 0, 2, 0, 5, 2, 3));
+ betweenChkp(2, 0, 0, 0, 0, 0, 2),
+ unmergedBcs(2, 0, 2, 0, 2, 2, 3),
+ btwnChkpUBC(2, 0, 2, 0, 2, 2, 3));
- assertNotExists("1:/a");
- assertNotExists("1:/b");
+ if (!isModeOneOf(DetailedGCMode.NONE, DetailedGCMode.GAP_ORPHANS,
DetailedGCMode.GAP_ORPHANS_EMPTYPROPS)) {
+ assertNotExists("1:/a");
+ assertNotExists("1:/b");
+ }
// now do another gc - should not have anything left to clean up though
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
stats = gc.gc(1, HOURS);
assertStatsCountsZero(stats);
- assertBranchRevisionRemovedFromAllDocuments(store, br1);
- assertBranchRevisionRemovedFromAllDocuments(store, br2);
+ if (!isModeOneOf(DetailedGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store, br2);
+ }
}
@Test
@@ -326,10 +364,14 @@ public class BranchCommitGCTest {
VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 1, 4,12, 0, 3),
keepOneUser(0, 0, 0, 4, 0, 0, 2),
- betweenChkp(0, 0, 1, 0,18, 2, 3));
+ betweenChkp(),
+ unmergedBcs(0, 0, 1, 0,16, 2, 3),
+ btwnChkpUBC(0, 0, 1, 0,16, 2, 3));
assertExists("1:/a");
assertExists("1:/b");
@@ -337,8 +379,10 @@ public class BranchCommitGCTest {
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
stats = gc.gc(1, HOURS);
assertStatsCountsZero(stats);
- assertBranchRevisionRemovedFromAllDocuments(store, br1);
- assertBranchRevisionRemovedFromAllDocuments(store, br2);
+ if (!isModeOneOf(DetailedGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store, br2);
+ }
}
@Test
@@ -382,10 +426,14 @@ public class BranchCommitGCTest {
VersionGarbageCollector.VersionGCStats stats = gc.gc(1, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(0, 0, 0, 0, 0, 0, 0),
+ gapOrphProp(0, 0, 0, 0, 0, 0, 0),
+ allOrphProp(0, 0, 0, 0, 0, 0, 0),
keepOneFull(0, 0, 1, 8,24, 0, 3),
keepOneUser(0, 0, 0, 8, 0, 0, 2),
- betweenChkp(0, 0, 1, 0,35, 4, 3));
+ betweenChkp(),
+ unmergedBcs(0, 0, 1, 0,32, 4, 3),
+ btwnChkpUBC(0, 0, 1, 0,32, 4, 3));
assertExists("1:/a");
assertExists("1:/b");
@@ -393,10 +441,12 @@ public class BranchCommitGCTest {
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
stats = gc.gc(1, HOURS);
assertStatsCountsZero(stats);
- assertBranchRevisionRemovedFromAllDocuments(store, br1);
- assertBranchRevisionRemovedFromAllDocuments(store, br2);
- assertBranchRevisionRemovedFromAllDocuments(store, br3);
- assertBranchRevisionRemovedFromAllDocuments(store, br4);
+ if (!isModeOneOf(DetailedGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store, br2);
+ assertBranchRevisionRemovedFromAllDocuments(store, br3);
+ assertBranchRevisionRemovedFromAllDocuments(store, br4);
+ }
}
@Test
@@ -418,17 +468,23 @@ public class BranchCommitGCTest {
// first gc round now deletes it, via orphaned node deletion
assertStatsCountsEqual(stats,
- noOldPropGc(1, 0, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(1, 0, 0, 0, 0, 0, 1),
keepOneFull(1, 0, 0, 1, 6, 0, 4),
keepOneUser(1, 0, 0, 1, 0, 0, 2),
- betweenChkp(1, 0, 0, 0, 7, 1, 4));
+ betweenChkp(1, 0, 0, 0, 0, 0, 1),
+ unmergedBcs(1, 0, 0, 0, 7, 1, 4),
+ btwnChkpUBC(1, 0, 0, 0, 7, 1, 4));
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// now do second gc round - should not have anything left to clean up
though
stats = gc.gc(1, HOURS);
assertStatsCountsZero(stats);
- assertBranchRevisionRemovedFromAllDocuments(store, br);
+ if (!isModeOneOf(DetailedGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
}
@Test
@@ -450,11 +506,17 @@ public class BranchCommitGCTest {
stats = gc.gc(1, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 0, 1, 4, 0, 2),
keepOneUser(0, 0, 0, 1, 0, 0, 1),
- betweenChkp(0, 0, 0, 1, 4, 1, 2));
- assertBranchRevisionRemovedFromAllDocuments(store, br);
+ betweenChkp(),
+ unmergedBcs(0, 0, 0, 1, 4, 1, 2),
+ btwnChkpUBC(0, 0, 0, 1, 4, 1, 2));
+ if (!isModeOneOf(DetailedGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
}
@Test
@@ -470,11 +532,17 @@ public class BranchCommitGCTest {
// 1 deleted prop: 1:/foo[a]
assertStatsCountsEqual(stats,
- noOldPropGc(0, 1, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
keepOneFull(0, 1, 0, 0, 4, 0, 2),
keepOneUser(0, 1, 0, 0, 0, 0, 1),
- betweenChkp(0, 1, 0, 0, 4, 1, 2));
- assertBranchRevisionRemovedFromAllDocuments(store, br);
+ betweenChkp(0, 1, 0, 0, 0, 0, 1),
+ unmergedBcs(0, 1, 0, 0, 4, 1, 2),
+ btwnChkpUBC(0, 1, 0, 0, 4, 1, 2));
+ if (!isModeOneOf(DetailedGCMode.NONE)) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
}
@Test
@@ -504,18 +572,24 @@ public class BranchCommitGCTest {
stats = gc.gc(1, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(0, 0, 0, 0, 0, 0, 0),
+ gapOrphProp(0, 0, 0, 0, 0, 0, 0),
+ allOrphProp(0, 0, 0, 0, 0, 0, 0),
keepOneFull(0, 0, 1,10,40, 0, 2),
keepOneUser(0, 0, 0,10, 0, 0, 1),
- betweenChkp(0, 0, 1, 0,59,10, 2));
+ betweenChkp(),
+ unmergedBcs(0, 0, 1, 0,50,10, 2),
+ btwnChkpUBC(0, 0, 1, 0,50,10, 2));
doc = store.getDocumentStore().find(Collection.NODES, "1:/foo");
Long finalModified = doc.getModified();
assertEquals(originalModified, finalModified);
- for (RevisionVector br : brs) {
- assertBranchRevisionRemovedFromAllDocuments(store, br);
+ if (!isModeOneOf(DetailedGCMode.NONE)) {
+ for (RevisionVector br : brs) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
}
}
@@ -547,12 +621,18 @@ public class BranchCommitGCTest {
stats = gc.gc(1, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(0, 0, 0, 0, 0, 0, 0),
+ gapOrphProp(0, 0, 0, 0, 0, 0, 0),
+ allOrphProp(0, 0, 0, 0, 0, 0, 0),
keepOneFull(0, 0, 2,10,30, 0, 2),
keepOneUser(0, 0, 0,10, 0, 0, 1),
- betweenChkp(0, 0, 2, 0,59,10, 2));
- for (RevisionVector br : brs) {
- assertBranchRevisionRemovedFromAllDocuments(store, br);
+ betweenChkp(),
+ unmergedBcs(0, 0, 2, 0,40,10, 2),
+ btwnChkpUBC(0, 0, 2, 0,40,10, 2));
+ if (!isModeOneOf(DetailedGCMode.NONE)) {
+ for (RevisionVector br : brs) {
+ assertBranchRevisionRemovedFromAllDocuments(store, br);
+ }
}
}
@@ -600,17 +680,26 @@ public class BranchCommitGCTest {
// deleted internal prop : 0:/ _collision
// deleted properties are 0:/ -> rootProp, _collisions & 1:/foo -> a
assertStatsCountsEqual(stats,
- noOldPropGc(0, 2, 0, 0, 0, 0, 2),
+ gapOrphOnly(),
+ gapOrphProp(0, 2, 0, 0, 0, 0, 2),
+ allOrphProp(0, 2, 0, 0, 0, 0, 2),
keepOneFull(0, 2, 1, 0, 8, 0, 2),
keepOneUser(0, 2, 0, 0, 0, 0, 2),
- betweenChkp(0, 2, 1, 0, 5, 1, 2));
+ betweenChkp(0, 2, 0, 0, 0, 0, 2),
+ unmergedBcs(0, 2, 1, 0, 4, 1, 2),
+ btwnChkpUBC(0, 2, 1, 0, 4, 1, 2));
+ if (isModeOneOf(DetailedGCMode.NONE, DetailedGCMode.GAP_ORPHANS)) {
+ return;
+ }
+
{
// some flakyness diagnostics
NodeDocument d = store.getDocumentStore().find(Collection.NODES,
"0:/", -1);
assertNotNull(d);
assertEquals(0, d.getLocalMap("rootProp").size());
- if (VersionGarbageCollector.getRevisionDetailedGcType() ==
RDGCType.KEEP_ONE_FULL_MODE
- || VersionGarbageCollector.getRevisionDetailedGcType() ==
RDGCType.OLDER_THAN_24H_AND_BETWEEN_CHECKPOINTS_MODE) {
+ if (VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS
+ || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.ORPHANS_EMPTYPROPS_UNMERGED_BC
+ || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC) {
assertEquals(0, d.getLocalMap("_collisions").size());
} else {
assertEquals(1, d.getLocalMap("_collisions").size());
diff --git
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java
index 6bbda403d6..ec7b792066 100644
---
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java
+++
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DetailGCHelper.java
@@ -18,7 +18,7 @@
*/
package org.apache.jackrabbit.oak.plugins.document;
-import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.RDGCType;
+import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.DetailedGCMode;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
@@ -74,7 +74,10 @@ public class DetailGCHelper {
final DocumentNodeStore store, final RevisionVector br,
final String... exceptIds) {
assertTrue(br.isBranch());
- if (VersionGarbageCollector.getRevisionDetailedGcType() ==
RDGCType.NO_OLD_PROP_REV_GC) {
+ if (VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.GAP_ORPHANS
+ || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.GAP_ORPHANS_EMPTYPROPS
+ || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.ALL_ORPHANS_EMPTYPROPS
+ || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC) {
// then we must skip these asserts, as we cannot guarantee
// that all revisions are cleaned up in this mode
return;
@@ -96,7 +99,7 @@ public class DetailGCHelper {
for (Entry<String, Object> e : target.data.entrySet()) {
String k = e.getKey();
final boolean internal = k.startsWith("_");
- final boolean dgcSupportsInternalPropCleanup =
(VersionGarbageCollector.getRevisionDetailedGcType() !=
RDGCType.KEEP_ONE_CLEANUP_USER_PROPERTIES_ONLY_MODE);
+ final boolean dgcSupportsInternalPropCleanup =
(VersionGarbageCollector.getDetailedGcMode() !=
DetailedGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS);
if (internal && !dgcSupportsInternalPropCleanup) {
// skip
continue;
diff --git
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
index 694124c468..5ba734fdf6 100644
---
a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
+++
b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
@@ -113,7 +113,7 @@ import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import
org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.RDBFixture;
import
org.apache.jackrabbit.oak.plugins.document.FailingDocumentStore.FailedUpdateOpListener;
-import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.RDGCType;
+import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.DetailedGCMode;
import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
import
org.apache.jackrabbit.oak.plugins.document.bundlor.BundlingConfigInitializer;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoTestUtils;
@@ -140,12 +140,16 @@ import org.junit.runners.Parameterized.Parameter;
public class VersionGarbageCollectorIT {
static class GCCounts {
- RDGCType mode;
+ private final DetailedGCMode mode;
int deletedDocGCCount, deletedPropsCount, deletedInternalPropsCount,
deletedPropRevsCount, deletedInternalPropRevsCount,
deletedUnmergedBCCount, updatedDetailedGCDocsCount;
- public GCCounts(RDGCType mode, int deletedDocGCCount, int
deletedPropsCount,
+ public GCCounts(DetailedGCMode mode) {
+ this(mode, 0, 0, 0, 0, 0, 0, 0);
+ }
+
+ public GCCounts(DetailedGCMode mode, int deletedDocGCCount, int
deletedPropsCount,
int deletedInternalPropsCount, int deletedPropRevsCount,
int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
int updatedDetailedGCDocsCount) {
@@ -157,11 +161,13 @@ public class VersionGarbageCollectorIT {
assertTrue(deletedInternalPropRevsCount != -1);
assertTrue(deletedUnmergedBCCount != -1);
assertTrue(updatedDetailedGCDocsCount != -1);
- if (mode == RDGCType.NO_OLD_PROP_REV_GC) {
+ if (mode == DetailedGCMode.GAP_ORPHANS
+ || mode == DetailedGCMode.GAP_ORPHANS_EMPTYPROPS
+ || mode == DetailedGCMode.ALL_ORPHANS_EMPTYPROPS) {
assertEquals(0, deletedPropRevsCount);
assertEquals(0, deletedInternalPropRevsCount);
assertEquals(0, deletedUnmergedBCCount);
- } else if (mode ==
RDGCType.KEEP_ONE_CLEANUP_USER_PROPERTIES_ONLY_MODE) {
+ } else if (mode ==
DetailedGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS) {
assertEquals(0, deletedInternalPropsCount);
assertEquals(0, deletedInternalPropRevsCount);
}
@@ -179,7 +185,7 @@ public class VersionGarbageCollectorIT {
public DocumentStoreFixture fixture;
@Parameter(1)
- public RDGCType gcType;
+ public DetailedGCMode detailedGcMode;
private Clock clock;
@@ -193,14 +199,14 @@ public class VersionGarbageCollectorIT {
private ExecutorService execService;
- private RDGCType originalGcType;
+ private DetailedGCMode originalDetailedGcMode;
@Parameterized.Parameters(name="{index}: {0} with {1}")
public static java.util.Collection<Object[]> params() throws IOException {
java.util.Collection<Object[]> params = new LinkedList<>();
for (Object[] fixture : AbstractDocumentStoreTest.fixtures()) {
DocumentStoreFixture f = (DocumentStoreFixture)fixture[0];
- for (RDGCType gcType : RDGCType.values()) {
+ for (DetailedGCMode gcType : DetailedGCMode.values()) {
params.add(new Object[] {f, gcType});
}
}
@@ -225,13 +231,13 @@ public class VersionGarbageCollectorIT {
MongoTestUtils.setReadPreference(store1, ReadPreference.primary());
gc = store1.getVersionGarbageCollector();
- originalGcType = VersionGarbageCollector.getRevisionDetailedGcType();
- writeStaticField(VersionGarbageCollector.class,
"revisionDetailedGcType", gcType, true);
+ originalDetailedGcMode = VersionGarbageCollector.getDetailedGcMode();
+ writeStaticField(VersionGarbageCollector.class, "detailedGcMode",
detailedGcMode, true);
}
@After
public void tearDown() throws Exception {
- writeStaticField(VersionGarbageCollector.class,
"revisionDetailedGcType", originalGcType, true);
+ writeStaticField(VersionGarbageCollector.class, "detailedGcMode",
originalDetailedGcMode, true);
if (store2 != null) {
store2.dispose();
}
@@ -414,7 +420,15 @@ public class VersionGarbageCollectorIT {
VersionGCStats stats = gc(gc, maxAge, TimeUnit.MILLISECONDS);
assertFalse("Detailed GC should be performed",
stats.ignoredDetailedGCDueToCheckPoint);
assertFalse(stats.canceled);
- assertEquals(batchSize, stats.updatedDetailedGCDocsCount);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, (int)batchSize, 0, 0, 0, 0, (int)batchSize),
+ allOrphProp(0, (int)batchSize, 0, 0, 0, 0, (int)batchSize),
+ keepOneFull(0, (int)batchSize, 0, 0, 0, 0, (int)batchSize),
+ keepOneUser(0, (int)batchSize, 0, 0, 0, 0, (int)batchSize),
+ unmergedBcs(0, (int)batchSize, 0, 0, 0, 0, (int)batchSize),
+ betweenChkp(0, (int)batchSize, 0, 0, 2, 0, (int)batchSize + 1),
+ btwnChkpUBC(0, (int)batchSize, 0, 0, 2, 0, (int)batchSize +
1));
assertFalse(stats.needRepeat);
}
@@ -485,10 +499,14 @@ public class VersionGarbageCollectorIT {
clock.waitUntil(clock.getTime() + delta*2);
VersionGCStats stats = gc(gc, delta, MILLISECONDS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 1, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
keepOneFull(0, 1, 0, 0, 0, 0, 1),
keepOneUser(0, 1, 0, 0, 0, 0, 1),
- betweenChkp(0, 1, 0, 0, 0, 0, 1));
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 0, 0, 1),
+ btwnChkpUBC(0, 1, 0, 0, 0, 0, 1));
assertTrue(stats.ignoredGCDueToCheckPoint);
assertTrue(stats.ignoredDetailedGCDueToCheckPoint);
assertTrue(stats.canceled);
@@ -541,10 +559,14 @@ public class VersionGarbageCollectorIT {
stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 1, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
keepOneFull(0, 1, 0, 0, 0, 0, 1),
keepOneUser(0, 1, 0, 0, 0, 0, 1),
- betweenChkp(0, 1, 0, 0, 0, 0, 1));
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 3, 0, 2),
+ btwnChkpUBC(0, 1, 0, 0, 3, 0, 2));
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
//4. Check that a revived property (deleted and created again) does
not get gc
@@ -560,10 +582,14 @@ public class VersionGarbageCollectorIT {
stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 0, 2, 0, 0, 1),
keepOneUser(0, 0, 0, 2, 0, 0, 1),
- betweenChkp(0, 0, 0, 0, 0, 0, 0));
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 1, 0, 0, 1),
+ btwnChkpUBC(0, 0, 0, 1, 0, 0, 1));
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
}
@@ -601,10 +627,14 @@ public class VersionGarbageCollectorIT {
VersionGCStats stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 50_000, 0, 0, 0, 0, 5_000),
+ gapOrphOnly(),
+ gapOrphProp(0, 50_000, 0, 0, 0, 0, 5_000),
+ allOrphProp(0, 50_000, 0, 0, 0, 0, 5_000),
keepOneFull(0, 50_000, 0, 0, 0, 0, 5_000),
keepOneUser(0, 50_000, 0, 0, 0, 0, 5_000),
- betweenChkp(0, 50_000, 0, 0, 0, 0, 5_000));
+ unmergedBcs(0, 50_000, 0, 0, 0, 0, 5_000),
+ betweenChkp(0, 50_000, 0, 0, 2, 0, 5_000 + 1),
+ btwnChkpUBC(0, 50_000, 0, 0, 2, 0, 5_000 + 1));
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
}
@@ -647,10 +677,14 @@ public class VersionGarbageCollectorIT {
VersionGCStats stats = gc(gc, maxAge, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 50_000, 0, 0, 0, 0, 5_000),
+ gapOrphOnly(),
+ gapOrphProp(0, 50_000, 0, 0, 0, 0, 5_000),
+ allOrphProp(0, 50_000, 0, 0, 0, 0, 5_000),
keepOneFull(0, 50_000, 0, 0, 0, 0, 5_000),
keepOneUser(0, 50_000, 0, 0, 0, 0, 5_000),
- betweenChkp(0, 50_000, 0, 0, 0, 0, 5_000));
+ unmergedBcs(0, 50_000, 0, 0, 0, 0, 5_000),
+ betweenChkp(0, 50_000, 0, 0, 51, 0, 5_000 + 1),
+ btwnChkpUBC(0, 50_000, 0, 0, 51, 0, 5_000 + 1));
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
}
@@ -727,10 +761,14 @@ public class VersionGarbageCollectorIT {
stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 10, 0, 0, 0, 0, 10),
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 10),
+ allOrphProp(0, 10, 0, 0, 0, 0, 10),
keepOneFull(0, 10, 0, 0, 0, 0, 10),
keepOneUser(0, 10, 0, 0, 0, 0, 10),
- betweenChkp(0, 10, 0, 0, 0, 0, 10));
+ unmergedBcs(0, 10, 0, 0, 0, 0, 10),
+ betweenChkp(0, 10, 0, 0, 2, 0, 11),
+ btwnChkpUBC(0, 10, 0, 0, 2, 0, 11));
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
//3. now reCreate those properties again
@@ -755,10 +793,14 @@ public class VersionGarbageCollectorIT {
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 10, 0, 0, 0, 0, 10),
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 10),
+ allOrphProp(0, 10, 0, 0, 0, 0, 10),
keepOneFull(0, 10, 0, 0, 0, 0, 10),
keepOneUser(0, 10, 0, 0, 0, 0, 10),
- betweenChkp(0, 10, 0, 0, 0, 0, 10));
+ unmergedBcs(0, 10, 0, 0, 0, 0, 10),
+ betweenChkp(0, 10, 0, 0, 2, 0, 11),
+ btwnChkpUBC(0, 10, 0, 0, 2, 0, 11));
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
}
@@ -829,10 +871,14 @@ public class VersionGarbageCollectorIT {
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
VersionGCStats stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 10, 0, 0, 0, 0, 10),
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 10),
+ allOrphProp(0, 10, 0, 0, 0, 0, 10),
keepOneFull(0, 10, 0, 0, 0, 0, 10),
keepOneUser(0, 10, 0, 0, 0, 0, 10),
- betweenChkp(0, 10, 0, 0, 0, 0, 10));
+ unmergedBcs(0, 10, 0, 0, 0, 0, 10),
+ betweenChkp(0, 10, 0, 0, 2, 0, 11),
+ btwnChkpUBC(0, 10, 0, 0, 2, 0, 11));
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
}
@@ -876,10 +922,14 @@ public class VersionGarbageCollectorIT {
stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 10, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 1),
+ allOrphProp(0, 10, 0, 0, 0, 0, 1),
keepOneFull(0, 10, 0, 0, 0, 0, 1),
keepOneUser(0, 10, 0, 0, 0, 0, 1),
- betweenChkp(0, 10, 0, 0, 0, 0, 1));
+ unmergedBcs(0, 10, 0, 0, 0, 0, 1),
+ betweenChkp(0, 10, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 10, 0, 0, 1, 0, 2));
//4. Check that a revived property (deleted and created again) does
not get gc
NodeBuilder b4 = store1.getRoot().builder();
@@ -890,7 +940,15 @@ public class VersionGarbageCollectorIT {
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
stats = gc(gc, maxAge*2, HOURS);
- assertStatsCountsZero(stats);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(),
+ keepOneUser(),
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
}
@Test
@@ -935,10 +993,14 @@ public class VersionGarbageCollectorIT {
stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 10, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 1),
+ allOrphProp(0, 10, 0, 0, 0, 0, 1),
keepOneFull(0, 10, 0, 0, 0, 0, 1),
keepOneUser(0, 10, 0, 0, 0, 0, 1),
- betweenChkp(0, 10, 0, 0, 0, 0, 1));
+ unmergedBcs(0, 10, 0, 0, 0, 0, 1),
+ betweenChkp(0, 10, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 10, 0, 0, 1, 0, 2));
//4. Check that a revived property (deleted and created again) does
not get gc
NodeBuilder b4 = store1.getRoot().builder();
@@ -949,7 +1011,15 @@ public class VersionGarbageCollectorIT {
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
stats = gc(gc, maxAge*2, HOURS);
- assertStatsCountsZero(stats);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
+ keepOneFull(),
+ keepOneUser(),
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
}
@Test
@@ -1003,10 +1073,14 @@ public class VersionGarbageCollectorIT {
stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 10, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 1),
+ allOrphProp(0, 10, 0, 0, 0, 0, 1),
keepOneFull(0, 10, 0, 0, 0, 0, 1),
keepOneUser(0, 10, 0, 0, 0, 0, 1),
- betweenChkp(0, 10, 0, 0, 0, 0, 1));
+ unmergedBcs(0, 10, 0, 0, 0, 0, 1),
+ betweenChkp(0, 10, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 10, 0, 0, 1, 0, 2));
}
@Test
@@ -1088,10 +1162,14 @@ public class VersionGarbageCollectorIT {
stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 10, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 10, 0, 0, 0, 0, 1),
+ allOrphProp(0, 10, 0, 0, 0, 0, 1),
keepOneFull(0, 10, 0, 0, 0, 0, 1),
keepOneUser(0, 10, 0, 0, 0, 0, 1),
- betweenChkp(0, 10, 0, 0, 0, 0, 1));
+ unmergedBcs(0, 10, 0, 0, 0, 0, 1),
+ betweenChkp(0, 10, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 10, 0, 0, 1, 0, 2));
x = store1.getRoot().getChildNode("x");
assertTrue(x.exists());
@@ -1104,28 +1182,30 @@ public class VersionGarbageCollectorIT {
}
static void assertStatsCountsZero(VersionGCStats stats) {
- GCCounts c = new
GCCounts(VersionGarbageCollector.getRevisionDetailedGcType(),
- 0, 0, 0, 0, 0, 0, 0);
+ GCCounts c = new GCCounts(VersionGarbageCollector.getDetailedGcMode());
assertStatsCountsEqual(stats, c);
}
static void assertStatsCountsEqual(VersionGCStats stats, GCCounts...
counts) {
GCCounts c = null;
for (GCCounts a : counts) {
- if (a.mode == VersionGarbageCollector.getRevisionDetailedGcType())
{
+ if (a.mode == VersionGarbageCollector.getDetailedGcMode()) {
c = a;
break;
}
}
+ if (c == null && VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.NONE) {
+ c = new GCCounts(DetailedGCMode.NONE);
+ }
assertNotNull(stats);
assertNotNull(c);
- assertEquals(c.deletedDocGCCount, stats.deletedDocGCCount);
- assertEquals(c.deletedPropsCount, stats.deletedPropsCount);
- assertEquals(c.deletedInternalPropsCount,
stats.deletedInternalPropsCount);
- assertEquals(c.deletedPropRevsCount, stats.deletedPropRevsCount);
- assertEquals(c.deletedInternalPropRevsCount,
stats.deletedInternalPropRevsCount);
- assertEquals(c.deletedUnmergedBCCount, stats.deletedUnmergedBCCount);
- assertEquals(c.updatedDetailedGCDocsCount,
stats.updatedDetailedGCDocsCount);
+ assertEquals(c.mode + "/docGC", c.deletedDocGCCount,
stats.deletedDocGCCount);
+ assertEquals(c.mode + "/props", c.deletedPropsCount,
stats.deletedPropsCount);
+ assertEquals(c.mode + "/internalProps", c.deletedInternalPropsCount,
stats.deletedInternalPropsCount);
+ assertEquals(c.mode + "/propRevs", c.deletedPropRevsCount,
stats.deletedPropRevsCount);
+ assertEquals(c.mode + "/internalPropRevs",
c.deletedInternalPropRevsCount, stats.deletedInternalPropRevsCount);
+ assertEquals(c.mode + "/unmergedBC", c.deletedUnmergedBCCount,
stats.deletedUnmergedBCCount);
+ assertEquals(c.mode + "/updatedDetailedGCDocsCount",
c.updatedDetailedGCDocsCount, stats.updatedDetailedGCDocsCount);
}
@Test
@@ -1324,10 +1404,14 @@ public class VersionGarbageCollectorIT {
assertNotNull(ckAfter);
assertEquals(ckBefore.getValue(Type.STRING),
ckAfter.getValue(Type.STRING));
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 0, 3, 0, 0, 2),
keepOneUser(0, 0, 0, 3, 0, 0, 2),
- betweenChkp(0, 0, 0, 0, 0, 0, 0));
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 1, 1, 0, 2),
+ btwnChkpUBC(0, 0, 0, 1, 1, 0, 2));
}
@Test
@@ -1359,10 +1443,14 @@ public class VersionGarbageCollectorIT {
// now the GC
VersionGCStats stats = gc(gc, 1, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 0, 1, 0, 0, 1),
keepOneUser(0, 0, 0, 1, 0, 0, 1),
- betweenChkp(0, 0, 0, 0, 0, 0, 0));
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
}
@Test
@@ -1428,8 +1516,7 @@ public class VersionGarbageCollectorIT {
// create branch commits
RevisionVector br1 = unmergedBranchCommit(store1, b ->
b.child("node1").setProperty("a", "2"));
store1.runBackgroundOperations();
- store1.invalidateNodeChildrenCache();
- store1.getNodeCache().invalidateAll();
+ invalidateCaches(store1);
assertEquals("1",
store1.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
// enable the detailed gc flag
@@ -1438,8 +1525,7 @@ public class VersionGarbageCollectorIT {
// wait two hours
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
- store1.invalidateNodeChildrenCache();
- store1.getNodeCache().invalidateAll();
+ invalidateCaches(store1);
assertEquals("1",
store1.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
// clean everything older than one hour
@@ -1449,37 +1535,36 @@ public class VersionGarbageCollectorIT {
nb.child("node1").setProperty("b", "4");
store1.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY);
VersionGCStats stats = gc(gc, 1, HOURS);
- store1.invalidateNodeChildrenCache();
- store1.getNodeCache().invalidateAll();
+ invalidateCaches(store1);
assertEquals("1",
store1.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
store1.runBackgroundOperations();
- store1.invalidateNodeChildrenCache();
- store1.getNodeCache().invalidateAll();
+ invalidateCaches(store1);
assertEquals("1",
store1.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
store1.runBackgroundOperations();
- store1.invalidateNodeChildrenCache();
- store1.getNodeCache().invalidateAll();
+ invalidateCaches(store1);
assertEquals("1",
store1.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
createSecondaryStore(LeaseCheckMode.LENIENT);
// while "2" was written to node1/a via an unmerged branch commit,
// it should not have been made visible through DGC/sweep combo
- store2.invalidateNodeChildrenCache();
- store2.getNodeCache().invalidateAll();
+ invalidateCaches(store2);
assertEquals("1",
store2.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
assertEquals("4",
store2.getRoot().getChildNode("node1").getProperty("b").getValue(Type.STRING));
- store2.invalidateNodeChildrenCache();
- store2.getNodeCache().invalidateAll();
+ invalidateCaches(store2);
assertEquals("1",
store2.getRoot().getChildNode("node1").getProperty("a").getValue(Type.STRING));
assertEquals("4",
store2.getRoot().getChildNode("node1").getProperty("b").getValue(Type.STRING));
// deletedPropsCount=0 : _bc on /node1 and / CANNOT be removed
// deletedPropRevsCount=1 : (nothing on /node1[a, _commitRoot),
/[_revisions]
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 1, 0, 1, 0, 1),
- keepOneUser(0, 0, 0, 0, 0, 0, 0),
- betweenChkp(0, 0, 1, 0, 2, 1, 1));
+ keepOneUser(),
+ unmergedBcs(0, 0, 1, 0, 1, 1, 1),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 1, 0, 2, 1, 1));
// checking for br1 revisino to have disappeared doesn't really make
much sense,
// since 1:/node1 isn't GCed as it is young, and 0:/ being root cannot
guarantee full removal
// (if br1 is deleted form 0:/ _bc, then the commit value resolution
flips it to committed)
@@ -1518,12 +1603,18 @@ public class VersionGarbageCollectorIT {
VersionGCStats stats = gc(gc, 1, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 3, 0, 0, 0, 0, 2),
+ gapOrphOnly(),
+ gapOrphProp(0, 3, 0, 0, 0, 0, 2),
+ allOrphProp(0, 3, 0, 0, 0, 0, 2),
keepOneFull(0, 3, 1, 1, 9, 0, 3),
keepOneUser(0, 3, 0, 1, 0, 0, 2),
- betweenChkp(0, 3, 1, 1, 8, 2, 3));
- assertBranchRevisionRemovedFromAllDocuments(store1, br1);
- assertBranchRevisionRemovedFromAllDocuments(store1, br4);
+ unmergedBcs(0, 3, 1, 1, 7, 2, 3),
+ betweenChkp(0, 3, 0, 0, 1, 0, 3),
+ btwnChkpUBC(0, 3, 1, 1, 8, 2, 3));
+ if (!isModeOneOf(DetailedGCMode.NONE, DetailedGCMode.GAP_ORPHANS,
DetailedGCMode.GAP_ORPHANS_EMPTYPROPS)) {
+ assertBranchRevisionRemovedFromAllDocuments(store1, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store1, br4);
+ }
}
@Test
@@ -1559,15 +1650,31 @@ public class VersionGarbageCollectorIT {
VersionGCStats stats = gc(gc, 1, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 3, 0, 0, 0, 0, 2),
+ gapOrphOnly(),
+ gapOrphProp(0, 3, 0, 0, 0, 0, 2),
+ allOrphProp(0, 3, 0, 0, 0, 0, 2),
keepOneFull(0, 3, 2, 1, 17, 0, 3),
keepOneUser(0, 3, 0, 1, 0, 0, 2),
- betweenChkp(0, 3, 2, 1, 18, 4, 3));
- assertBranchRevisionRemovedFromAllDocuments(store1, br1);
- assertBranchRevisionRemovedFromAllDocuments(store1, br2);
- assertBranchRevisionRemovedFromAllDocuments(store1, br3);
- assertBranchRevisionRemovedFromAllDocuments(store1, br4);
+ unmergedBcs(0, 3, 2, 1, 15, 4, 3),
+ betweenChkp(0, 3, 0, 0, 1, 0, 3),
+ btwnChkpUBC(0, 3, 2, 1, 16, 4, 3));
+ if (!isModeOneOf(DetailedGCMode.NONE, DetailedGCMode.GAP_ORPHANS,
DetailedGCMode.GAP_ORPHANS_EMPTYPROPS)) {
+ assertBranchRevisionRemovedFromAllDocuments(store1, br1);
+ assertBranchRevisionRemovedFromAllDocuments(store1, br2);
+ assertBranchRevisionRemovedFromAllDocuments(store1, br3);
+ assertBranchRevisionRemovedFromAllDocuments(store1, br4);
+ }
+ }
+
+ static boolean isModeOneOf(DetailedGCMode... modes) {
+ for (DetailedGCMode rdgcType : modes) {
+ if (VersionGarbageCollector.getDetailedGcMode() == rdgcType) {
+ return true;
+ }
+ }
+ return false;
}
+
// OAK-8646 END
/**
@@ -1575,7 +1682,15 @@ public class VersionGarbageCollectorIT {
*/
@Test
public void lateWriteCreateChildGC() throws Exception {
- doLateWriteCreateChildrenGC(of("/grand/parent"),
of("/grand/parent/a"), 1, "/d");
+ doLateWriteCreateChildrenGC(of("/grand/parent"),
of("/grand/parent/a"), "/d",
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(1, 0, 0, 0, 0, 0, 1),
+ keepOneFull(1, 0, 0, 0, 0, 0, 1),
+ keepOneUser(1, 0, 0, 0, 0, 0, 1),
+ unmergedBcs(1, 0, 0, 0, 0, 0, 1),
+ betweenChkp(1, 0, 0, 0, 2, 0, 2),
+ btwnChkpUBC(1, 0, 0, 0, 2, 0, 2));
}
/**
@@ -1584,7 +1699,15 @@ public class VersionGarbageCollectorIT {
*/
@Test
public void lateWriteCreateChildTreeGC() throws Exception {
- doLateWriteCreateChildrenGC(of("/a", "/a/b/c"), of("/a/b/c/d",
"/a/b/c/d/e/f"), 3, "/d");
+ doLateWriteCreateChildrenGC(of("/a", "/a/b/c"), of("/a/b/c/d",
"/a/b/c/d/e/f"), "/d",
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(3, 0, 0, 0, 0, 0, 3),
+ keepOneFull(3, 0, 0, 0, 0, 0, 3),
+ keepOneUser(3, 0, 0, 0, 0, 0, 3),
+ unmergedBcs(3, 0, 0, 0, 0, 0, 3),
+ betweenChkp(3, 0, 0, 0, 3/*4*/, 0, 4),
+ btwnChkpUBC(3, 0, 0, 0, 3/*4*/, 0, 4));
}
/**
@@ -1595,7 +1718,15 @@ public class VersionGarbageCollectorIT {
public void lateWriteCreateChildGCLargePath() throws Exception {
String longPath = repeat("p", PATH_LONG + 1);
String path = "/grand/parent/" + longPath + "/longPathChild";
- doLateWriteCreateChildrenGC(of("/grand/parent"), of(path), 2, "/d");
+ doLateWriteCreateChildrenGC(of("/grand/parent"), of(path), "/d",
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(2, 0, 0, 0, 0, 0, 2),
+ keepOneFull(2, 0, 0, 0, 0, 0, 2),
+ keepOneUser(2, 0, 0, 0, 0, 0, 2),
+ unmergedBcs(2, 0, 0, 0, 0, 0, 2),
+ betweenChkp(2, 0, 0, 0, 2/*3*/, 0, 3),
+ btwnChkpUBC(2, 0, 0, 0, 2/*3*/, 0, 3));
}
/**
@@ -1614,7 +1745,18 @@ public class VersionGarbageCollectorIT {
commonOrphanParents.add(orphanParent);
orphans.add(orphanParent + "/" + r.nextInt(24));
}
- doLateWriteCreateChildrenGC(nonOrphans, orphans, orphans.size() +
commonOrphanParents.size(), "/d");
+ int expectedNumOrphanedDocs = orphans.size() +
commonOrphanParents.size();
+ int expectedNumInternalPropRevsGCed = 4;//904=expectedNumOrphanedDocs
+ 1;
+ int expectedNumDocsUpdatedGCed = expectedNumOrphanedDocs + 1;
+ doLateWriteCreateChildrenGC(nonOrphans, orphans, "/d",
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(expectedNumOrphanedDocs, 0, 0, 0, 0, 0,
expectedNumOrphanedDocs),
+ keepOneFull(expectedNumOrphanedDocs, 0, 0, 0, 0, 0,
expectedNumOrphanedDocs),
+ keepOneUser(expectedNumOrphanedDocs, 0, 0, 0, 0, 0,
expectedNumOrphanedDocs),
+ unmergedBcs(expectedNumOrphanedDocs, 0, 0, 0, 0, 0,
expectedNumOrphanedDocs),
+ betweenChkp(expectedNumOrphanedDocs, 0, 0, 0,
expectedNumInternalPropRevsGCed, 0, expectedNumDocsUpdatedGCed),
+ btwnChkpUBC(expectedNumOrphanedDocs, 0, 0, 0,
expectedNumInternalPropRevsGCed, 0, expectedNumDocsUpdatedGCed));
}
@Test
@@ -1637,10 +1779,14 @@ public class VersionGarbageCollectorIT {
assertTrue(getChildeNodeState(store1, "/a/b/c/d", true).exists());
// should be 3 as it should clean up the _deleted from /a/b, /a/b/c
and /a/b/c/d
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 0, 3, 3, 0, 3),
keepOneUser(0, 0, 0, 3, 0, 0, 3),
- betweenChkp(0, 0, 0, 0, 0, 0, 0));
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
}
/**
@@ -1703,10 +1849,24 @@ public class VersionGarbageCollectorIT {
VersionGCStats stats = gc(store1.getVersionGarbageCollector(), 1,
HOURS);
assertNotNull(stats);
// expected 2 updated (deletions): /a/b/c/d and /a/b/c/d/e
- assertEquals(2, stats.updatedDetailedGCDocsCount);
- assertEquals(2, stats.deletedDocGCCount);
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(2, 0, 0, 0, 0, 0, 2),
+ gapOrphProp(2, 0, 0, 0, 0, 0, 2),
+ allOrphProp(2, 0, 0, 0, 0, 0, 2),
+ keepOneFull(2, 0, 0, 0, 0, 0, 2),
+ keepOneUser(2, 0, 0, 0, 0, 0, 2),
+ unmergedBcs(2, 0, 0, 0, 0, 0, 2),
+ betweenChkp(2, 0, 0, 0, 3, 0, 4),
+ btwnChkpUBC(2, 0, 0, 0, 3, 0, 4));
- createNodes("/a/b/c/d/e");
+ if (isModeOneOf(DetailedGCMode.NONE)) {
+ // in these modes the inconsistency isn't cleaned up
+ // which means there will be a OakMerge0004 exception upon
+ // trying to create the node(s) again.
+ // hence we can't really do this in thsse modes
+ } else {
+ createNodes("/a/b/c/d/e");
+ }
}
@Ignore(value="this is a reminder to add bundling-detailedGC tests in
general, plus some of those cases combined with OAK-10542")
@@ -1758,13 +1918,57 @@ public class VersionGarbageCollectorIT {
// below is an example of how the different modes result in different
cleanups
// this might help us narrow down differences in the modes
assertStatsCountsEqual(stats,
- noOldPropGc(0, 2, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 2, 0, 0, 0, 0, 1),
+ allOrphProp(0, 2, 0, 0, 0, 0, 1),
keepOneFull(0, 2, 2, 3, 11, 0, 2),
keepOneUser(0, 2, 0, 3, 0, 0, 1),
- betweenChkp(0, 2, 1, 2, 13, 3, 2));
+ unmergedBcs(0, 2, 1, 2, 12, 3, 2),
+ betweenChkp(0, 2, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 2, 1, 2, 13, 3, 2));
+ }
+
+ static GCCounts gapOrphOnly() {
+ return new GCCounts(DetailedGCMode.GAP_ORPHANS);
+ }
+
+ static GCCounts gapOrphOnly(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedDetailedGCDocsCount) {
+ assertEquals(0, deletedInternalPropsCount);
+ assertEquals(0, deletedPropRevsCount);
+ assertEquals(0, deletedInternalPropRevsCount);
+ assertEquals(0, deletedUnmergedBCCount);
+ return new GCCounts(DetailedGCMode.GAP_ORPHANS, deletedDocGCCount,
+ deletedPropsCount, deletedInternalPropsCount,
deletedPropRevsCount,
+ deletedInternalPropRevsCount, deletedUnmergedBCCount,
+ updatedDetailedGCDocsCount);
+ }
+
+ static GCCounts gapOrphProp() {
+ return new GCCounts(DetailedGCMode.GAP_ORPHANS_EMPTYPROPS);
+ }
+
+ static GCCounts gapOrphProp(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedDetailedGCDocsCount) {
+ assertEquals(0, deletedInternalPropsCount);
+ assertEquals(0, deletedPropRevsCount);
+ assertEquals(0, deletedInternalPropRevsCount);
+ assertEquals(0, deletedUnmergedBCCount);
+ return new GCCounts(DetailedGCMode.GAP_ORPHANS_EMPTYPROPS,
deletedDocGCCount,
+ deletedPropsCount, deletedInternalPropsCount,
deletedPropRevsCount,
+ deletedInternalPropRevsCount, deletedUnmergedBCCount,
+ updatedDetailedGCDocsCount);
}
- static GCCounts noOldPropGc(int deletedDocGCCount, int deletedPropsCount,
+ static GCCounts allOrphProp() {
+ return new GCCounts(DetailedGCMode.ALL_ORPHANS_EMPTYPROPS);
+ }
+
+ static GCCounts allOrphProp(int deletedDocGCCount, int deletedPropsCount,
int deletedInternalPropsCount, int deletedPropRevsCount,
int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
int updatedDetailedGCDocsCount) {
@@ -1772,37 +1976,77 @@ public class VersionGarbageCollectorIT {
assertEquals(0, deletedPropRevsCount);
assertEquals(0, deletedInternalPropRevsCount);
assertEquals(0, deletedUnmergedBCCount);
- return new GCCounts(RDGCType.NO_OLD_PROP_REV_GC, deletedDocGCCount,
+ return new GCCounts(DetailedGCMode.ALL_ORPHANS_EMPTYPROPS,
deletedDocGCCount,
deletedPropsCount, deletedInternalPropsCount,
deletedPropRevsCount,
deletedInternalPropRevsCount, deletedUnmergedBCCount,
updatedDetailedGCDocsCount);
}
+ static GCCounts keepOneFull() {
+ return new
GCCounts(DetailedGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS);
+ }
+
static GCCounts keepOneFull(int deletedDocGCCount, int deletedPropsCount,
int deletedInternalPropsCount, int deletedPropRevsCount,
int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
int updatedDetailedGCDocsCount) {
- return new GCCounts(RDGCType.KEEP_ONE_FULL_MODE, deletedDocGCCount,
+ return new
GCCounts(DetailedGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_ALL_PROPS,
deletedDocGCCount,
deletedPropsCount, deletedInternalPropsCount,
deletedPropRevsCount,
deletedInternalPropRevsCount, deletedUnmergedBCCount,
updatedDetailedGCDocsCount);
}
+ static GCCounts keepOneUser() {
+ return new
GCCounts(DetailedGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS);
+ }
+
static GCCounts keepOneUser(int deletedDocGCCount, int deletedPropsCount,
int deletedInternalPropsCount, int deletedPropRevsCount,
int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
int updatedDetailedGCDocsCount) {
- return new
GCCounts(RDGCType.KEEP_ONE_CLEANUP_USER_PROPERTIES_ONLY_MODE,
+ return new
GCCounts(DetailedGCMode.ORPHANS_EMPTYPROPS_KEEP_ONE_USER_PROPS,
deletedDocGCCount, deletedPropsCount,
deletedInternalPropsCount,
deletedPropRevsCount, deletedInternalPropRevsCount,
deletedUnmergedBCCount, updatedDetailedGCDocsCount);
}
+ static GCCounts unmergedBcs() {
+ return new GCCounts(DetailedGCMode.ORPHANS_EMPTYPROPS_UNMERGED_BC);
+ }
+
+ static GCCounts unmergedBcs(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedDetailedGCDocsCount) {
+ return new GCCounts(DetailedGCMode.ORPHANS_EMPTYPROPS_UNMERGED_BC,
+ deletedDocGCCount, deletedPropsCount,
deletedInternalPropsCount,
+ deletedPropRevsCount, deletedInternalPropRevsCount,
+ deletedUnmergedBCCount, updatedDetailedGCDocsCount);
+ }
+
+ static GCCounts betweenChkp() {
+ return new
GCCounts(DetailedGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC);
+ }
+
static GCCounts betweenChkp(int deletedDocGCCount, int deletedPropsCount,
int deletedInternalPropsCount, int deletedPropRevsCount,
int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
int updatedDetailedGCDocsCount) {
- return new
GCCounts(RDGCType.OLDER_THAN_24H_AND_BETWEEN_CHECKPOINTS_MODE,
+ return new
GCCounts(DetailedGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC,
+ deletedDocGCCount, deletedPropsCount,
deletedInternalPropsCount,
+ deletedPropRevsCount, deletedInternalPropRevsCount,
+ deletedUnmergedBCCount, updatedDetailedGCDocsCount);
+ }
+
+ static GCCounts btwnChkpUBC() {
+ return new
GCCounts(DetailedGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC);
+ }
+
+ static GCCounts btwnChkpUBC(int deletedDocGCCount, int deletedPropsCount,
+ int deletedInternalPropsCount, int deletedPropRevsCount,
+ int deletedInternalPropRevsCount, int deletedUnmergedBCCount,
+ int updatedDetailedGCDocsCount) {
+ return new
GCCounts(DetailedGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC,
deletedDocGCCount, deletedPropsCount,
deletedInternalPropsCount,
deletedPropRevsCount, deletedInternalPropRevsCount,
deletedUnmergedBCCount, updatedDetailedGCDocsCount);
@@ -1854,10 +2098,14 @@ public class VersionGarbageCollectorIT {
clock.waitUntil(getCurrentTimestamp() + maxAgeMillis + 1);
VersionGCStats stats = gc(gc, maxAgeHours, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 0, 11, 0, 0, 1),
keepOneUser(0, 0, 0, 11, 0, 0, 1),
- betweenChkp(0, 0, 0, 0, 0, 0, 0));
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
NodeState x = store1.getRoot().getChildNode("x");
assertTrue(x.exists());
@@ -1870,8 +2118,12 @@ public class VersionGarbageCollectorIT {
NodeDocument doc = store1.getDocumentStore().find(NODES, "1:/x", -1);
assertNotNull(doc);
- if (VersionGarbageCollector.getRevisionDetailedGcType() ==
RDGCType.OLDER_THAN_24H_AND_BETWEEN_CHECKPOINTS_MODE
- || VersionGarbageCollector.getRevisionDetailedGcType() ==
RDGCType.NO_OLD_PROP_REV_GC) {
+ if (VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_WITH_UNMERGED_BC
+ || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.NONE || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.GAP_ORPHANS
+ || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.GAP_ORPHANS_EMPTYPROPS
+ || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.ALL_ORPHANS_EMPTYPROPS
+ || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.ORPHANS_EMPTYPROPS_UNMERGED_BC
+ || VersionGarbageCollector.getDetailedGcMode() ==
DetailedGCMode.ORPHANS_EMPTYPROPS_BETWEEN_CHECKPOINTS_NO_UNMERGED_BC) {
// this mode doesn't currently delete all revisions,
// thus would fail below assert.
return;
@@ -1912,10 +2164,14 @@ public class VersionGarbageCollectorIT {
VersionGCStats stats = gc(gc, maxAge*2, HOURS);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 1, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
keepOneFull(0, 1, 0, 0, 0, 0, 1),
keepOneUser(0, 1, 0, 0, 0, 0, 1),
- betweenChkp(0, 1, 0, 0, 0, 0, 1));
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 1, 0, 0, 1, 0, 2));
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
// 4. Save values of detailedGC settings collection fields
@@ -1941,10 +2197,15 @@ public class VersionGarbageCollectorIT {
final String oldestModifiedDryRunDocId = stats.oldestModifiedDocId;
final long oldestModifiedDocDryRunTimeStamp =
stats.oldestModifiedDocTimeStamp;
- assertStatsCountsEqual(stats, noOldPropGc(0, 1, 0, 0, 0, 0, 1),
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
keepOneFull(0, 1, 0, 0, 0, 0, 1),
keepOneUser(0, 1, 0, 0, 0, 0, 1),
- betweenChkp(0, 1, 0, 0, 0, 0, 1));
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 1, 0, 0, 1, 0, 2));
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
assertTrue(stats.detailedGCDryRunMode);
@@ -1989,10 +2250,15 @@ public class VersionGarbageCollectorIT {
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
VersionGCStats stats = gc(gc, 1, HOURS);
- assertStatsCountsEqual(stats, noOldPropGc(0, 3, 0, 0, 0, 0, 2),
- keepOneFull(0, 3, 2, 1, 17, 0, 3),
+ assertStatsCountsEqual(stats,
+ gapOrphOnly(),
+ gapOrphProp(0, 3, 0, 0, 0, 0, 2),
+ allOrphProp(0, 3, 0, 0, 0, 0, 2),
+ keepOneFull(0, 3, 2, 1,17, 0, 3),
keepOneUser(0, 3, 0, 1, 0, 0, 2),
- betweenChkp(0, 3, 2, 1, 18, 4, 3));
+ unmergedBcs(0, 3, 2, 1,15, 4, 3),
+ betweenChkp(0, 3, 0, 0, 1, 0, 3),
+ btwnChkpUBC(0, 3, 2, 1,16, 4, 3));
assertTrue(stats.detailedGCDryRunMode);
assertBranchRevisionNotRemovedFromAllDocuments(store1, br1);
@@ -2035,10 +2301,14 @@ public class VersionGarbageCollectorIT {
assertFalse(store1.getRoot().getChildNode("bar").hasProperty("prop"));
assertNotNull(stats);
assertStatsCountsEqual(stats,
- noOldPropGc(0, 1, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
keepOneFull(0, 1, 0, 0, 0, 0, 1),
keepOneUser(0, 1, 0, 0, 0, 0, 1),
- betweenChkp(0, 1, 0, 0, 0, 0, 1));
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 1, 0, 2),
+ btwnChkpUBC(0, 1, 0, 0, 1, 0, 2));
assertDocumentsExist(of("/bar"));
}
@@ -2076,10 +2346,14 @@ public class VersionGarbageCollectorIT {
// we should still be seeing the garbage from late write and
// thus it will be collected.
assertStatsCountsEqual(stats,
- noOldPropGc(0, 1, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
keepOneFull(0, 1, 0, 0, 0, 0, 1),
keepOneUser(0, 1, 0, 0, 0, 0, 1),
- betweenChkp(0, 1, 0, 0, 0, 0, 1));
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ betweenChkp(0, 1, 0, 0, 2, 0, 2),
+ btwnChkpUBC(0, 1, 0, 0, 2, 0, 2));
assertDocumentsExist(of("/foo/bar/baz"));
}
@@ -2119,10 +2393,14 @@ public class VersionGarbageCollectorIT {
// deletedPropRevsCount : 2 prop-revs GCed : the original prop=value,
plus the
// removeProperty(prop) plus 1 _commitRoot entry
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 0, 2, 0, 0, 1),
keepOneUser(0, 0, 0, 2, 0, 0, 1),
- betweenChkp(0, 0, 0, 0, 0, 0, 0));
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 2, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 2, 0, 1));
assertDocumentsExist(of("/bar"));
}
@@ -2137,7 +2415,7 @@ public class VersionGarbageCollectorIT {
// unrelated path should be such that the paths and unrelated path
shouldn't have common parent
// for e.g. if path is /bar & unrelated is /d then there common
ancestor is "/" i.e. root.
- lateWriteRemovePropertiesNodes(of("/bar"), null, "p");
+ lateWriteRemovePropertiesNodes(of("/bar"), null, false, "p");
assertDocumentsExist(of("/bar"));
assertPropertyNotExist("/bar", "p");
@@ -2151,10 +2429,14 @@ public class VersionGarbageCollectorIT {
assertNotNull(stats);
// 1 prop-rev removal : the late-write null
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 0, 1, 0, 0, 1),
keepOneUser(0, 0, 0, 1, 0, 0, 1),
- betweenChkp(0, 0, 0, 0, 0, 0, 0));
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 1, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 1, 0, 1));
assertDocumentsExist(of("/bar"));
}
@@ -2169,7 +2451,7 @@ public class VersionGarbageCollectorIT {
// unrelated path should be such that the paths and unrelated path
shouldn't have common parent
// for e.g. if path is /bar & unrelated is /d then there common
ancestor is "/" i.e. root.
- lateWriteRemovePropertiesNodes(of("/foo/bar/baz"), "/a", "prop");
+ lateWriteRemovePropertiesNodes(of("/foo/bar/baz"), "/a", false,
"prop");
assertDocumentsExist(of("/foo/bar/baz"));
assertPropertyNotExist("/foo/bar/baz", "prop");
@@ -2185,15 +2467,41 @@ public class VersionGarbageCollectorIT {
// thus it will be collected.
// removes 1 prop-rev : the late-write null
assertStatsCountsEqual(stats,
- noOldPropGc(0, 0, 0, 0, 0, 0, 0),
+ gapOrphOnly(),
+ gapOrphProp(),
+ allOrphProp(),
keepOneFull(0, 0, 0, 1, 0, 0, 1),
keepOneUser(0, 0, 0, 1, 0, 0, 1),
- betweenChkp(0, 0, 0, 0, 0, 0, 0));
+ unmergedBcs(),
+ betweenChkp(0, 0, 0, 0, 2, 0, 1),
+ btwnChkpUBC(0, 0, 0, 0, 2, 0, 1));
assertDocumentsExist(of("/foo/bar/baz"));
+ invalidateCaches(store1);
+ assertEquals("value",
store1.getRoot().getChildNode("foo").getChildNode("bar")
+
.getChildNode("baz").getProperty("prop").getValue(Type.STRING));
+ }
+
+ @SuppressWarnings("unchecked")
+ private void invalidateCaches(DocumentNodeStore dns) throws
IllegalAccessException {
+ dns.invalidateNodeChildrenCache();
+ dns.getNodeCache().invalidateAll();
+ dns.getNodeChildrenCache().invalidateAll();
+ CachingCommitValueResolver cvr = (CachingCommitValueResolver)
readField(dns, "commitValueResolver", true);
+ Cache<Revision, String> commitValueCache = (Cache<Revision, String>)
readField( cvr, "commitValueCache", true);
+ commitValueCache.invalidateAll();
+ }
+
+ @Test
+ public void skipPropertyRemovedByLateWriteWithRelatedPath_normal() throws
Exception {
+ doSkipPropertyRemovedByLateWriteWithRelatedPath(false);
}
@Test
- public void skipPropertyRemovedByLateWriteWithRelatedPath() throws
Exception {
+ public void skipPropertyRemovedByLateWriteWithRelatedPath_branch() throws
Exception {
+ doSkipPropertyRemovedByLateWriteWithRelatedPath(true);
+ }
+
+ private void doSkipPropertyRemovedByLateWriteWithRelatedPath(boolean
useBranchForUnrelatedPath) throws Exception {
// create a node with property.
assumeTrue(fixture.hasSinglePersistence());
NodeBuilder nb = store1.getRoot().builder();
@@ -2203,7 +2511,7 @@ public class VersionGarbageCollectorIT {
// unrelated path should be such that the paths and unrelated path
shouldn't have common parent
// for e.g. if path is /bar & unrelated is /d then there common
ancestor is "/" i.e. root.
- lateWriteRemovePropertiesNodes(of("/bar"), "/d", "prop");
+ lateWriteRemovePropertiesNodes(of("/bar"), "/d",
useBranchForUnrelatedPath, "prop");
assertDocumentsExist(of("/bar"));
assertPropertyNotExist("/bar", "prop");
@@ -2214,15 +2522,25 @@ public class VersionGarbageCollectorIT {
clock.waitUntil(clock.getTime() + HOURS.toMillis(2));
// clean everything older than one hour
VersionGCStats stats = gc(store1.getVersionGarbageCollector(), 1,
HOURS);
+ assertTrue(store1.getRoot().getChildNode("d").exists());
+ invalidateCaches(store1);
+ assertTrue(store1.getRoot().getChildNode("d").exists());
assertNotNull(stats);
// we should be able to remove the property since we have updated an
related path that has lead to an update
// of common ancestor and this would make late write visible
assertStatsCountsEqual(stats,
- noOldPropGc(0, 1, 0, 0, 0, 0, 1),
+ gapOrphOnly(),
+ gapOrphProp(0, 1, 0, 0, 0, 0, 1),
+ allOrphProp(0, 1, 0, 0, 0, 0, 1),
keepOneFull(0, 1, 0, 0, 0, 0, 1),
keepOneUser(0, 1, 0, 0, 0, 0, 1),
- betweenChkp(0, 1, 0, 0, 0, 0, 1));
-
+ unmergedBcs(0, 1, 0, 0, 0, 0, 1),
+ useBranchForUnrelatedPath ?
+ betweenChkp(0, 1, 0, 0, 1, 0, 2) :
+ betweenChkp(0, 1, 0, 0, 2, 0, 2),
+ useBranchForUnrelatedPath ?
+ btwnChkpUBC(0, 1, 0, 0, 1, 0, 2) :
+ btwnChkpUBC(0, 1, 0, 0, 2, 0, 2));
assertDocumentsExist(of("/bar"));
}
// OAK-10676 END
@@ -3283,26 +3601,32 @@ public class VersionGarbageCollectorIT {
private void lateWriteCreateNodes(Collection<String> orphanedPaths,
String unrelatedPathOrNull) throws
Exception {
- lateWrite(orphanedPaths, TestUtils::createChild,
- unrelatedPathOrNull, ADD_NODE_OPS, (ds, ops) ->
ds.createOrUpdate(NODES, ops));
+ lateWrite(orphanedPaths, TestUtils::createChild, unrelatedPathOrNull,
false,
+ ADD_NODE_OPS, (ds, ops) -> ds.createOrUpdate(NODES, ops));
}
private void lateWriteRemoveNodes(Collection<String> orphanedPaths,
String unrelatedPathOrNull) throws
Exception {
lateWrite(orphanedPaths, (rb, path) -> childBuilder(rb, path).remove(),
- unrelatedPathOrNull, REMOVE_NODE_OPS, (ds, ops) ->
ds.createOrUpdate(NODES, ops));
+ unrelatedPathOrNull, false, REMOVE_NODE_OPS,
+ (ds, ops) -> ds.createOrUpdate(NODES, ops));
}
private void lateWriteAddPropertiesNodes(Collection<String> paths, String
unrelatedPath, String propertyName,
String propertyValue) throws
Exception {
- lateWrite(paths, (rb, path) -> childBuilder(rb,
path).setProperty(propertyName, propertyValue), unrelatedPath,
- setPropertyOps(propertyName), (ds, ops) ->
ds.findAndUpdate(NODES, ops));
+ lateWrite(paths,
+ (rb, path) -> childBuilder(rb, path).setProperty(propertyName,
+ propertyValue),
+ unrelatedPath, false, setPropertyOps(propertyName),
+ (ds, ops) -> ds.findAndUpdate(NODES, ops));
}
- private void lateWriteRemovePropertiesNodes(Collection<String> paths,
String unrelatedPath, String propertyName)
+ private void lateWriteRemovePropertiesNodes(Collection<String> paths,
String unrelatedPath, boolean bc4Unrelated, String propertyName)
throws Exception {
- lateWrite(paths, (rb, path) -> childBuilder(rb,
path).removeProperty(propertyName), unrelatedPath,
- setPropertyOps(propertyName), (ds, ops) ->
ds.findAndUpdate(NODES, ops));
+ lateWrite(paths,
+ (rb, path) -> childBuilder(rb,
path).removeProperty(propertyName),
+ unrelatedPath, bc4Unrelated, setPropertyOps(propertyName),
+ (ds, ops) -> ds.findAndUpdate(NODES, ops));
}
/**
@@ -3317,7 +3641,7 @@ public class VersionGarbageCollectorIT {
* @param dataStoreConsumer persist late write changes to DocumentStore
* @throws Exception in case of merge failure we throw exception
*/
- private void lateWrite(Collection<String> paths, LateWriteChangesBuilder
lateWriteChangesBuilder, String unrelatedPath,
+ private void lateWrite(Collection<String> paths, LateWriteChangesBuilder
lateWriteChangesBuilder, String unrelatedPath, boolean bc4Unrealted,
Predicate<UpdateOp> filterPredicate,
BiConsumer<DocumentStore, List<UpdateOp>> dataStoreConsumer) throws Exception {
// this method requires store2 to be null as a prerequisite
assertNull(store2);
@@ -3360,7 +3684,11 @@ public class VersionGarbageCollectorIT {
// revive clusterId 2
createSecondaryStore(LeaseCheckMode.LENIENT);
- merge(store2, createChild(store2.getRoot().builder(), unrelatedPath));
+ if (bc4Unrealted) {
+ mergedBranchCommit(store2, nb -> createChild(nb, unrelatedPath));
+ } else {
+ merge(store2, createChild(store2.getRoot().builder(),
unrelatedPath));
+ }
store2.runBackgroundOperations();
store2.dispose();
store1.runBackgroundOperations();
@@ -3382,7 +3710,7 @@ public class VersionGarbageCollectorIT {
* root to allow detecting late-writes as
such
*/
private void doLateWriteCreateChildrenGC(Collection<String> parents,
- Collection<String> orphans, int
expectedNumOrphanedDocs, String unrelatedPath)
+ Collection<String> orphans, String unrelatedPath, GCCounts...
counts)
throws Exception {
assumeTrue(fixture.hasSinglePersistence());
createNodes(parents);
@@ -3399,12 +3727,13 @@ public class VersionGarbageCollectorIT {
// clean everything older than one hour
VersionGCStats stats = gc(store1.getVersionGarbageCollector(), 1,
HOURS);
assertNotNull(stats);
- assertEquals(expectedNumOrphanedDocs, stats.deletedDocGCCount);
-
+ assertStatsCountsEqual(stats, counts);
assertDocumentsExist(parents);
// and the main assert being: have those lateCreated (orphans) docs
been deleted
assertNodesDontExist(parents, orphans);
- assertDocumentsDontExist(orphans);
+ if (!isModeOneOf(DetailedGCMode.NONE, DetailedGCMode.GAP_ORPHANS,
DetailedGCMode.GAP_ORPHANS_EMPTYPROPS)) {
+ assertDocumentsDontExist(orphans);
+ }
}
private void assertNodesDontExist(Collection<String> existingNodes,
