This is an automated email from the ASF dual-hosted git repository. joscorbe pushed a commit to branch old-revisions-cleanup-rebase in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 9af3b34a0eecc869a30560b74f4e9c8fda0638df Author: Jose Cordero <[email protected]> AuthorDate: Mon Jan 29 19:39:34 2024 +0100 Some fixes and refactor. This needs a big cleanup (WIP) --- .../document/DocumentRevisionCleanupHelper.java | 4 +- .../document/NodeDocumentRevisionCleaner.java | 320 ++++++++++++++++++++ .../plugins/document/VersionGarbageCollector.java | 33 ++- .../DocumentRevisionCleanupHelperTest.java | 84 +++++- ...t.java => NodeDocumentRevisionCleanerTest.java} | 323 +++++++++++++++++---- 5 files changed, 688 insertions(+), 76 deletions(-) diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelper.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelper.java index efb5f7a31a..0613444efe 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelper.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelper.java @@ -18,11 +18,11 @@ */ package org.apache.jackrabbit.oak.plugins.document; -import com.google.common.collect.Maps; import org.apache.jackrabbit.oak.plugins.document.util.Utils; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.util.HashMap; import java.util.Map; import java.util.NavigableMap; import java.util.SortedMap; @@ -251,7 +251,7 @@ public class DocumentRevisionCleanupHelper { * @return */ public Map<Integer, Revision> getSweepRev() { - Map<Integer, Revision> map = Maps.newHashMap(); + Map<Integer, Revision> map = new HashMap<>(); Map<Revision, String> valueMap = (SortedMap<Revision, String>) rootDoc.get("_sweepRev"); for (Map.Entry<Revision, String> e : valueMap.entrySet()) { int clusterId = e.getKey().getClusterId(); diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java new file mode 100644 index 0000000000..ed28c957ed --- /dev/null +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.document; + +import org.apache.jackrabbit.oak.plugins.document.util.Utils; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.HashMap; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.TreeSet; + +import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES; + +public class NodeDocumentRevisionCleaner { + + private final DocumentStore documentStore; + private final DocumentNodeStore documentNodeStore; + private final NodeDocument workingDocument; + + protected RevisionClassifierUtility revisionClassifier; + protected RevisionCleanerUtility revisionCleaner; + + /** + * Constructor for DocumentRevisionCleanupHelper. + * @param documentStore The DocumentStore instance. Must be writable to perform cleanup. + * @param documentNodeStore The DocumentNodeStore instance. + * @param path The path of the document to clean up. + */ + public NodeDocumentRevisionCleaner(DocumentStore documentStore, DocumentNodeStore documentNodeStore, String path) { + String id = Utils.getIdFromPath(path); + this.workingDocument = documentStore.find(NODES, id); + this.documentStore = documentStore; + this.documentNodeStore = documentNodeStore; + + revisionClassifier = new RevisionClassifierUtility(workingDocument); + revisionCleaner = new RevisionCleanerUtility(revisionClassifier); + } + + /** + * Performs the full revision cleanup process for the given document for a clusterId. + */ + public void initializeCleanupProcess() { + revisionClassifier.identifyRevisionsToClean(); + revisionCleaner.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS); + revisionCleaner.markLastRevisionForEachProperty(); + revisionCleaner.markCheckpointRevisionsToPreserve(); + revisionCleaner.removeCandidatesInList(); + } + + public int executeCleanupProcess(int numberToCleanup, int clusterToCleanup) { + return -99; + } + + protected void classifyAndMapRevisionsAndProperties() { + //identifyRevisionsToClean(); + //BclassifyAndMapRevisionsAndProperties(); + + revisionClassifier.identifyRevisionsToClean(); + //newClassifyAndMapRevisionsAndProperties(); + } + + /*protected void newClassifyAndMapRevisionsAndProperties() { + candidateRevisionsToClean = new TreeMap<>(); + blockedRevisionsToKeep = new TreeMap<>(); + revisionsModifyingPropertyByCluster = new TreeMap<>(); + revisionsModifyingProperty = new TreeMap<>(); + propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE); + + // The first entry in "_deleted" has to be kept, as is when the document was created + //NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.get("_deleted")); + SortedMap<Revision, String> deletedRevisions = workingDocument.getLocalDeleted(); + if (deletedRevisions != null && !deletedRevisions.isEmpty()) { + Revision createdRevision = deletedRevisions.firstKey(); + addBlockedRevisionToKeep(createdRevision); + } + + SortedMap<Revision, String> documentRevisions = getAllRevisions(); + for (Map.Entry<Revision, String> revisionEntry : documentRevisions.entrySet()) { + Revision revision = revisionEntry.getKey(); + String revisionValue = revisionEntry.getValue(); + + // Only check committed revisions (ignore branch commits starting with "c-") + if ("c".equals(revisionValue)) { + // Candidate to clean up + addCandidateRevisionToClean(revision); + // Store properties usage + mapPropertiesModifiedByThisRevision(revision); + } + } + }*/ + + /** + * Step 1: + * This method processes the revisions of the working document, classifying them into two categories: + * candidate revisions that can be cleaned up and used revisions that should be kept. It also creates maps to + * track the relationships between revisions and properties modified by them. + */ + /*protected void BclassifyAndMapRevisionsAndProperties() { + candidateRevisionsToClean = new TreeMap<>(); + blockedRevisionsToKeep = new TreeMap<>(); + revisionsModifyingPropertyByCluster = new TreeMap<>(); + revisionsModifyingProperty = new TreeMap<>(); + propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE); + + // The first entry in "_deleted" has to be kept, as is when the document was created + //NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.get("_deleted")); + NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.getLocalDeleted()); + if (deletedRevisions != null && !deletedRevisions.isEmpty()) { + // TODO: This is just a check to ensure the order is the expected + assert(deletedRevisions.descendingMap().lastKey().getTimestamp() <= deletedRevisions.descendingMap().firstKey().getTimestamp()); + + Revision createdRevision = deletedRevisions.descendingMap().lastKey(); + addBlockedRevisionToKeep(createdRevision); + } + + SortedMap<Revision, String> documentRevisions = getAllRevisions(); + for (Map.Entry<Revision, String> revisionEntry : documentRevisions.entrySet()) { + Revision revision = revisionEntry.getKey(); + String revisionValue = revisionEntry.getValue(); + + // Only check committed revisions (ignore branch commits starting with "c-") + if ("c".equals(revisionValue)) { + // Candidate to clean up + addCandidateRevisionToClean(revision); + // Store properties usage + mapPropertiesModifiedByThisRevision(revision); + } + } + }*/ + + protected class RevisionClassifierUtility { + private final NodeDocument workingDocument; + private final SortedMap<Revision, String> documentRevisions; + private SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingPropertyByCluster; + private SortedMap<String, TreeSet<Revision>> revisionsModifyingProperty; + private SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision; + + RevisionClassifierUtility(NodeDocument workingDocument) { + this.workingDocument = workingDocument; + this.documentRevisions = workingDocument.getLocalRevisions(); + + this.revisionsModifyingPropertyByCluster = new TreeMap<>(); + this.revisionsModifyingProperty = new TreeMap<>(); + this.propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE); + } + + public void identifyRevisionsToClean() { + SortedMap<Revision, String> deletedRevisions = workingDocument.getLocalDeleted(); + // Always keep the first "_deleted" entry, as is when the document was created + if (!deletedRevisions.isEmpty()) { + Revision createdRevision = deletedRevisions.firstKey(); + revisionCleaner.addBlockedRevisionToKeep(createdRevision); + } + + SortedMap<Revision, String> documentRevisions = workingDocument.getLocalRevisions(); + for (Map.Entry<Revision, String> revisionEntry : documentRevisions.entrySet()) { + Revision revision = revisionEntry.getKey(); + String revisionValue = revisionEntry.getValue(); + + // Only check committed revisions (ignore branch commits starting with "c-") + if (Utils.isCommitted(revisionValue)) { + // Candidate to clean up + revisionCleaner.addCandidateRevisionToClean(revision); + // Store properties usage + mapPropertiesModifiedByThisRevision(revision); + } + } + } + + private void mapPropertiesModifiedByThisRevision(Revision revision) { + for (Map.Entry<String, Object> propertyEntry : workingDocument.entrySet()) { + if (Utils.isPropertyName(propertyEntry.getKey()) || NodeDocument.isDeletedEntry(propertyEntry.getKey())) { + Map<Revision, String> valueMap = (Map) propertyEntry.getValue(); + if (valueMap.containsKey(revision)) { + propertiesModifiedByRevision.computeIfAbsent(revision, key -> + new TreeSet<>()).add(propertyEntry.getKey() + ); + + revisionsModifyingPropertyByCluster.computeIfAbsent(propertyEntry.getKey(), key -> + new TreeMap<>() + ).computeIfAbsent(revision.getClusterId(), key -> + new TreeSet<>(StableRevisionComparator.INSTANCE) + ).add(revision); + + revisionsModifyingProperty.computeIfAbsent(propertyEntry.getKey(), key -> + new TreeSet<>(StableRevisionComparator.INSTANCE) + ).add(revision); + } + } + } + } + } + + protected class RevisionCleanerUtility { + + private SortedMap<Integer, TreeSet<Revision>> blockedRevisionsToKeep; + private SortedMap<Integer, TreeSet<Revision>> candidateRevisionsToClean; + private final RevisionClassifierUtility revisionClassifier; + + protected RevisionCleanerUtility(RevisionClassifierUtility revisionClassifier) { + this.revisionClassifier = revisionClassifier; + this.candidateRevisionsToClean = new TreeMap<>(); + this.blockedRevisionsToKeep = new TreeMap<>(); + } + + protected void markLastRevisionForEachProperty() { + for (SortedMap<Integer, TreeSet<Revision>> revisionsByCluster : revisionClassifier.revisionsModifyingPropertyByCluster.values()) { + for (TreeSet<Revision> revisions : revisionsByCluster.values()) { + Revision lastRevision = revisions.last(); + addBlockedRevisionToKeep(lastRevision); + } + } + } + + protected void markRevisionsNewerThanThresholdToPreserve(long amount, ChronoUnit unit) { + long thresholdToPreserve = Instant.now().minus(amount, unit).toEpochMilli(); + for (TreeSet<Revision> revisionSet : candidateRevisionsToClean.values()) { + for (Revision revision : revisionSet) { + if (revision.getTimestamp() > thresholdToPreserve) { + addBlockedRevisionToKeep(revision); + } + } + } + } + + protected void markCheckpointRevisionsToPreserve() { + SortedMap<Revision, Checkpoints.Info> checkpoints = documentNodeStore.getCheckpoints().getCheckpoints(); + checkpoints.forEach((revision, info) -> { + // For each checkpoint, keep the last revision that modified a property prior to checkpoint + revisionClassifier.revisionsModifyingProperty.forEach((propertyName, revisionsSet) -> { + // Traverse the revisionVector of the checkpoint and find the last revision that modified the property + info.getCheckpoint().forEach(revisionToFind -> { + // If the exact revision exists, keep it. If not, find the previous one that modified that property + if (revisionsSet.contains(revisionToFind)) { + addBlockedRevisionToKeep(revisionToFind); + } else { + Revision previousRevision = revisionsSet.descendingSet().ceiling(revisionToFind); + if (previousRevision != null) { + addBlockedRevisionToKeep(previousRevision); + } + } + }); + }); + }); + } + + /** + * Adds a revision to the list of candidates to delete. + * @param revision + */ + private void addCandidateRevisionToClean(Revision revision) { + candidateRevisionsToClean.computeIfAbsent(revision.getClusterId(), key -> + new TreeSet<>(StableRevisionComparator.INSTANCE) + ).add(revision); + } + + /** + * Adds a revision to the list of revisions to keep. + * @param revision + */ + private void addBlockedRevisionToKeep(Revision revision) { + blockedRevisionsToKeep.computeIfAbsent(revision.getClusterId(), key -> + new TreeSet<>(StableRevisionComparator.INSTANCE) + ).add(revision); + } + + protected void removeCandidatesInList() { + revisionCleaner.blockedRevisionsToKeep.forEach((key, value) -> { + if (revisionCleaner.candidateRevisionsToClean.containsKey(key)) { + revisionCleaner.candidateRevisionsToClean.get(key).removeAll(value); + } + }); + } + } + + public NavigableMap<Revision, String> getAllRevisions() { + return (NavigableMap<Revision, String>) workingDocument.getLocalRevisions(); + } + + public SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> getRevisionsModifyingPropertyByCluster() { + return revisionClassifier.revisionsModifyingPropertyByCluster; + } + + public SortedMap<String, TreeSet<Revision>> getRevisionsModifyingProperty() { + return revisionClassifier.revisionsModifyingProperty; + } + + public SortedMap<Revision, TreeSet<String>> getPropertiesModifiedByRevision() { + return revisionClassifier.propertiesModifiedByRevision; + } + + public SortedMap<Integer, TreeSet<Revision>> getBlockedRevisionsToKeep() { + return revisionCleaner.blockedRevisionsToKeep; + } + + public SortedMap<Integer, TreeSet<Revision>> getCandidateRevisionsToClean() { + return revisionCleaner.candidateRevisionsToClean; + } +} diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java index 73565b29fc..d9fa8e3205 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java @@ -944,7 +944,7 @@ public class VersionGarbageCollector { collectDeletedProperties(doc, phases, op); collectUnmergedBranchCommits(doc, phases, op, toModifiedMs); - collectOldRevisions(doc, phases, op); + collectOldRevisions(doc, phases, op, toModifiedMs); // only add if there are changes for this doc if (op.hasChanges()) { garbageDocsCount++; @@ -1053,6 +1053,28 @@ public class VersionGarbageCollector { phases.stop(GCPhase.DETAILED_GC_COLLECT_UNMERGED_BC); } + private void collectOldRevisions(final NodeDocument doc, final GCPhases phases, final UpdateOp updateOp, + final long toModifiedMs) { + if (phases.start(GCPhase.DETAILED_GC_COLLECT_OLD_REVS)){ + // Identify the revisions that can be removed + final Set<Revision> oldRevisions = doc.getLocalRevisions().keySet().stream() + .filter(rev -> isRevisionOlderThan(rev, toModifiedMs)) + .collect(toSet()); + + if (oldRevisions.isEmpty()) { + phases.stop(GCPhase.DETAILED_GC_COLLECT_OLD_REVS); + return; + } + + + + + + + phases.stop(GCPhase.DETAILED_GC_COLLECT_OLD_REVS); + } + } + /** * Filter all would be empty system properties (after cleanup operation). * <p> @@ -1189,15 +1211,6 @@ public class VersionGarbageCollector { }; } - private void collectOldRevisions(final NodeDocument doc, final GCPhases phases, final UpdateOp updateOp) { - - if (phases.start(GCPhase.DETAILED_GC_COLLECT_OLD_REVS)){ - // TODO add old rev collection logic - phases.stop(GCPhase.DETAILED_GC_COLLECT_OLD_REVS); - } - - } - int getGarbageCount() { return totalGarbageDocsCount; } diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java index b56df5ae2c..e1f6608c9d 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java @@ -1,5 +1,9 @@ package org.apache.jackrabbit.oak.plugins.document; +import org.apache.jackrabbit.guava.common.collect.Maps; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; @@ -7,10 +11,12 @@ import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import java.io.IOException; +import java.lang.reflect.Field; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -24,6 +30,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES; +import static org.apache.jackrabbit.oak.plugins.document.TestUtils.merge; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; @@ -43,14 +50,17 @@ public class DocumentRevisionCleanupHelperTest { @Mock NodeDocument workingDocument; + //NodeDocumentRevisionCleaner NodeDocumentRevisionCleaner; DocumentRevisionCleanupHelper documentRevisionCleanupHelper; @Before public void setUp() { MockitoAnnotations.initMocks(this); workingDocument = Mockito.mock(NodeDocument.class); + //workingDocument = new NodeDocument(documentStore); Mockito.when(documentStore.find(Mockito.eq(NODES), Mockito.anyString())).thenReturn(workingDocument); + //NodeDocumentRevisionCleaner = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, "/"); documentRevisionCleanupHelper = new DocumentRevisionCleanupHelper(documentStore, documentNodeStore, "/"); } @@ -317,7 +327,7 @@ public class DocumentRevisionCleanupHelperTest { } @Test - public void testFirstDeletedRevisionIsBlocked() throws IOException { + public void testFirstDeletedRevisionIsBlocked() throws Exception { Revision revisionA = new Revision(111111111L, 0, 1); Revision revisionB = new Revision(222222222L, 0, 1); Revision revisionC = new Revision(333333333L, 0, 1); @@ -334,6 +344,78 @@ public class DocumentRevisionCleanupHelperTest { assertEquals(Set.of(revisionA, revisionB, revisionC), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); } + /*@Test + public void testFirstDeletedRevisionIsBlocked() throws Exception { + Revision revisionA = new Revision(111111111L, 0, 1); + Revision revisionB = new Revision(222222222L, 0, 1); + Revision revisionC = new Revision(333333333L, 0, 1); + + String jsonProperties = "{" + + "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," + + "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" + + "}"; + //prepareDocumentMock(jsonProperties); + + insertJsonDataToNodeDocument(workingDocument, jsonProperties); + + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + + assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); + assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + }*/ + + /*private void insertJsonDataToNodeDocument(NodeDocument document, String jsonProperties) throws Exception { + String json = jsonProperties.replace("'", "\""); + + ObjectMapper mapper = new ObjectMapper(); + // TypeReference helps in specifying the complex type (Map<String, Object>) + Map<String, Object> parsedData = mapper.readValue(json, new TypeReference<Map<String, Object>>() {}); + + // Since the data field is a Map<String, Object>, we need to ensure the inner maps are also correctly typed + for (Map.Entry<String, Object> entry : parsedData.entrySet()) { + if (entry.getValue() instanceof Map) { + Map<?, ?> innerMap = (Map<?, ?>) entry.getValue(); + Map<String, Object> correctedInnerMap = new TreeMap<>(); + for (Map.Entry<?, ?> innerEntry : innerMap.entrySet()) { + correctedInnerMap.put(innerEntry.getKey().toString(), innerEntry.getValue()); + } + parsedData.put(entry.getKey(), correctedInnerMap); + } + } + + document.data = parsedData; // Directly setting the protected field + }*/ + + /*@Test + public void testtest() throws CommitFailedException, NoSuchFieldException, IllegalAccessException { + DocumentStore store = new MemoryDocumentStore(); + DocumentNodeStore ns = new DocumentMK.Builder().setDocumentStore(store).setAsyncDelay(0).getNodeStore(); + + // add properties + for (int i = 0; i < 10; i++) { + NodeBuilder nb = ns.getRoot().builder(); + + nb.child("x").setProperty("p"+i, i); + merge(ns, nb); + } + + NodeDocument nodeDocument = store.find(NODES, "1:/x"); + System.out.println(nodeDocument.getPropertyNames()); + + // Mocked data + Map<String, Object> data = Maps.newHashMap(); + + + NodeDocument document = new NodeDocument(store, System.currentTimeMillis()); + document.data.put("foo", "bar"); + System.out.println(document.getPropertyNames()); + + document.data = Maps.newHashMap(); + + + System.out.println(document.getPropertyNames()); + }*/ + @Test public void testClassifyAndMapRevisionsAndPropertiesWithDeleted() throws IOException { Revision revisionA = new Revision(111111111L, 0, 1); diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java similarity index 60% copy from oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java copy to oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java index b56df5ae2c..4cd9731c21 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java @@ -1,5 +1,8 @@ package org.apache.jackrabbit.oak.plugins.document; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; @@ -20,16 +23,13 @@ import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -public class DocumentRevisionCleanupHelperTest { +public class NodeDocumentRevisionCleanerTest { @Mock DocumentStore documentStore; @@ -40,18 +40,25 @@ public class DocumentRevisionCleanupHelperTest { @Mock Checkpoints checkpoints; - @Mock + //@Mock NodeDocument workingDocument; - DocumentRevisionCleanupHelper documentRevisionCleanupHelper; + NodeDocumentRevisionCleaner NodeDocumentRevisionCleaner; @Before public void setUp() { MockitoAnnotations.initMocks(this); - workingDocument = Mockito.mock(NodeDocument.class); + //workingDocument = Mockito.mock(NodeDocument.class); + + /*documentStore = new MemoryDocumentStore(); + documentNodeStore = new DocumentMK.Builder() + .setDocumentStore(documentStore).setAsyncDelay(0).setClusterId(1).build();*/ + + workingDocument = new NodeDocument(documentStore); Mockito.when(documentStore.find(Mockito.eq(NODES), Mockito.anyString())).thenReturn(workingDocument); - documentRevisionCleanupHelper = new DocumentRevisionCleanupHelper(documentStore, documentNodeStore, "/"); + Mockito.when(documentNodeStore.getBranches()).thenReturn(new UnmergedBranches()); + NodeDocumentRevisionCleaner = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, "/"); } @Test @@ -79,15 +86,15 @@ public class DocumentRevisionCleanupHelperTest { prepareDocumentMock(jsonProperties); prepareCheckpointsMock(jsonCheckpoints); - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - documentRevisionCleanupHelper.markCheckpointRevisionsToPreserve(); - documentRevisionCleanupHelper.removeCandidatesInList(documentRevisionCleanupHelper.getBlockedRevisionsToKeep()); + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + NodeDocumentRevisionCleaner.revisionCleaner.markCheckpointRevisionsToPreserve(); + NodeDocumentRevisionCleaner.revisionCleaner.removeCandidatesInList(); // The revisions blocked should be: // - r105000000-0-1 (blocked by checkpoint r109000000-0-1) // - r115000000-0-1 (blocked by checkpoint r119000000-0-1) - assertEquals(Set.of(revisionB, revisionD), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revisionA, revisionC, revisionE, revisionF), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); + assertEquals(Set.of(revisionB, revisionD), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); + assertEquals(Set.of(revisionA, revisionC, revisionE, revisionF), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); } @Test @@ -129,9 +136,9 @@ public class DocumentRevisionCleanupHelperTest { prepareDocumentMock(jsonBuilder.toString()); - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - documentRevisionCleanupHelper.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS); - documentRevisionCleanupHelper.removeCandidatesInList(documentRevisionCleanupHelper.getBlockedRevisionsToKeep()); + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + NodeDocumentRevisionCleaner.revisionCleaner.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS); + NodeDocumentRevisionCleaner.revisionCleaner.removeCandidatesInList(); // The candidate revisions should be the 6 oldest ones (current time -29, -28, -27, -26, -25, -24) // and the one created 24 hours and 1 minute ago @@ -142,8 +149,8 @@ public class DocumentRevisionCleanupHelperTest { // Blocked revisions are all the 24 revisions created in the last 24 hours, and the one created 23 hours and 59 minutes ago Set<Revision> expectedBlockedRevisions = revisions.subList(6, 31).stream().collect(Collectors.toSet()); - assertEquals(expectedCandidateRevisions, documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); - assertEquals(expectedBlockedRevisions, documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1)); + assertEquals(expectedCandidateRevisions, NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertEquals(expectedBlockedRevisions, NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); } @Test @@ -214,24 +221,24 @@ public class DocumentRevisionCleanupHelperTest { prepareDocumentMock(jsonBuilder.toString()); prepareCheckpointsMock(jsonCheckpoints); - documentRevisionCleanupHelper.initializeCleanupProcess(); + NodeDocumentRevisionCleaner.initializeCleanupProcess(); // The revisions blocked should be: // - r106000003-0-3, r118000004-0-2, r108000005-0-3 (referenced by checkpoint 1) // - r118000009-1-1, r118000010-0-2, r118000011-0-3 (referenced by checkpoint 2) // - r130000016-1-1, r130000017-0-2, r130000018-0-3 (last revision) - assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(2)); - assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(3)); + assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); + assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2)); + assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3)); // Rest of revisions are candidates to clean - assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); - assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(2)); - assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(3)); + assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)); + assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)); - assertTrue(Collections.disjoint(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1))); - assertTrue(Collections.disjoint(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(2), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(2))); - assertTrue(Collections.disjoint(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(3), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(3))); + assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1))); + assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2))); + assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3))); } @Test @@ -280,21 +287,120 @@ public class DocumentRevisionCleanupHelperTest { prepareDocumentMock(jsonBuilder.toString()); prepareCheckpointsMock(jsonCheckpoints); - documentRevisionCleanupHelper.initializeCleanupProcess(); + NodeDocumentRevisionCleaner.initializeCleanupProcess(); // The revisions blocked should be: // - r103000008-0-1, r103000009-0-2, r103000010-0-3 (last revisions) // - r102000006-0-1, r101300005-0-3, r102000007-0-3 (referenced by checkpoint for clusters 1, 2 and 3 respectively) - assertEquals(Set.of(revs.get(6), revs.get(8)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revs.get(9)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(2)); - assertEquals(Set.of(revs.get(5), revs.get(7), revs.get(10)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(3)); + assertEquals(Set.of(revs.get(6), revs.get(8)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); + assertEquals(Set.of(revs.get(9)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2)); + assertEquals(Set.of(revs.get(5), revs.get(7), revs.get(10)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3)); // Rest of revisions are candidates to clean - assertEquals(Set.of(revs.get(0), revs.get(3)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); - assertEquals(Set.of(revs.get(1), revs.get(4)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(2)); - assertEquals(Set.of(revs.get(2)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(3)); + assertEquals(Set.of(revs.get(0), revs.get(3)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertEquals(Set.of(revs.get(1), revs.get(4)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)); + assertEquals(Set.of(revs.get(2)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)); } + // New test copy + /*@Test + public void testInitializeCleanupProcessMultipleClustersNew() throws IOException { + List<Revision> revs = new ArrayList<>(); + + // Some initial revisions (0-2) + revs.add(Revision.fromString("r100000000-0-1")); + revs.add(Revision.fromString("r100000001-0-2")); + revs.add(Revision.fromString("r100000002-0-3")); + + // Blocked by first checkpoint r109000000 (3-5) + revs.add(Revision.fromString("r106000003-0-1")); + revs.add(Revision.fromString("r107000004-0-2")); + revs.add(Revision.fromString("r108000005-0-3")); + + // Some unblocked revisions in middle (6-10) + revs.add(Revision.fromString("r110000006-0-3")); + revs.add(Revision.fromString("r110000006-1-3")); + revs.add(Revision.fromString("r114000007-0-2")); + revs.add(Revision.fromString("r115000008-0-1")); + revs.add(Revision.fromString("r118000009-0-1")); + + // Blocked by second checkpoint r118000000 (11-13) + revs.add(Revision.fromString("r118000009-1-1")); + revs.add(Revision.fromString("r118000010-0-2")); + revs.add(Revision.fromString("r118000011-0-3")); + + // Some more unblocked revisions (14-19) + revs.add(Revision.fromString("r120000012-0-1")); + revs.add(Revision.fromString("r122000013-0-2")); + revs.add(Revision.fromString("r122000013-1-2")); + revs.add(Revision.fromString("r123000014-0-2")); + revs.add(Revision.fromString("r125000015-0-3")); + revs.add(Revision.fromString("r130000016-0-1")); + + // Last revision (20-22) + revs.add(Revision.fromString("r130000016-1-1")); + revs.add(Revision.fromString("r130000017-0-2")); + revs.add(Revision.fromString("r130000018-0-3")); + + // Checkpoint revisions + Revision checkpoint1 = Revision.fromString("r109000000-0-1"); + Revision checkpoint2 = Revision.fromString("r119000000-0-1"); + + StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {"); + StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {"); + + for (int i = 0; i < revs.size(); i++) { + jsonPropBuilder.append("'").append(revs.get(i)).append("': 'value").append(i).append("'"); + jsonRevisionsBuilder.append("'").append(revs.get(i)).append("': 'c'"); + if (i < revs.size() - 1) { + jsonPropBuilder.append(", "); + jsonRevisionsBuilder.append(", "); + } + } + + jsonPropBuilder.append("}"); + jsonRevisionsBuilder.append("}"); + StringBuilder jsonBuilder = new StringBuilder("{"); + jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append(", '_deleted': {'r100000000-0-1': 'false'}").append("}"); + + String jsonCheckpoints = "{" + + "'" + checkpoint1 + "': {'expires':'200000000','rv':'r106500003-0-1,r107500004-0-2,r108500005-0-3'}," + + "'" + checkpoint2 + "': {'expires':'200000000','rv':'r118000009-1-1,r118000010-0-2,r118000011-0-3'}" + + "}"; + + prepareDocumentMock(jsonBuilder.toString()); + prepareCheckpointsMock(jsonCheckpoints); + + NodeDocumentRevisionCleaner.initializeCleanupProcess(); + + RevisionVector readRevision1 = documentNodeStore.getCheckpoints().getCheckpoints().get(checkpoint1).getCheckpoint(); + RevisionVector readRevision2 = documentNodeStore.getCheckpoints().getCheckpoints().get(checkpoint2).getCheckpoint(); + + //workingDocument.getLiveRevision(); + DocumentNodeState state1 = workingDocument.getNodeAtRevision(documentNodeStore, readRevision1,null); + DocumentNodeState state2 = workingDocument.getNodeAtRevision(documentNodeStore, readRevision2,null); + + System.out.println(state1); + System.out.println(state2); + + // The revisions blocked should be: + // - r106000003-0-3, r118000004-0-2, r108000005-0-3 (referenced by checkpoint 1) + // - r118000009-1-1, r118000010-0-2, r118000011-0-3 (referenced by checkpoint 2) + // - r130000016-1-1, r130000017-0-2, r130000018-0-3 (last revision) + /*assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); + assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2)); + assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3)); + + // Rest of revisions are candidates to clean + assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)); + assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)); + + assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1))); + assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2))); + assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)));*/ + //} + @Test public void testLastRevisionIsBlocked() throws IOException { Revision revisionA = new Revision(111111111L, 0, 1); @@ -307,17 +413,17 @@ public class DocumentRevisionCleanupHelperTest { "}"; prepareDocumentMock(jsonProperties); - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - documentRevisionCleanupHelper.markLastRevisionForEachProperty(); + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + NodeDocumentRevisionCleaner.revisionCleaner.markLastRevisionForEachProperty(); - assertFalse(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1).contains(revisionA)); - assertFalse(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1).contains(revisionB)); - assertEquals(Set.of(revisionC), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revisionA, revisionB, revisionC), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); + assertFalse(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); + assertFalse(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionB)); + assertEquals(Set.of(revisionC), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); + assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); } @Test - public void testFirstDeletedRevisionIsBlocked() throws IOException { + public void testFirstDeletedRevisionIsBlocked() throws Exception { Revision revisionA = new Revision(111111111L, 0, 1); Revision revisionB = new Revision(222222222L, 0, 1); Revision revisionC = new Revision(333333333L, 0, 1); @@ -328,12 +434,102 @@ public class DocumentRevisionCleanupHelperTest { "}"; prepareDocumentMock(jsonProperties); - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); - assertTrue(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1).contains(revisionA)); - assertEquals(Set.of(revisionA, revisionB, revisionC), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); + assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); + assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); } + @Test + public void testFirstDeletedRevisionIsBlockedUnordered() throws Exception { + Revision revisionA = new Revision(111111111L, 0, 1); + Revision revisionB = new Revision(222222222L, 0, 1); + Revision revisionC = new Revision(333333333L, 0, 1); + + String jsonProperties = "{" + + "'_deleted': {'" + revisionC + "': 'false', '" + revisionA + "': 'true', '" + revisionB + "': 'false'}," + + "'_revisions': {'" + revisionC + "': 'c', '" + revisionB + "': 'c', '" + revisionA + "': 'c'}" + + "}"; + prepareDocumentMock(jsonProperties); + + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + + assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); + assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + } + + /*@Test + public void testFirstDeletedRevisionIsBlocked() throws Exception { + Revision revisionA = new Revision(111111111L, 0, 1); + Revision revisionB = new Revision(222222222L, 0, 1); + Revision revisionC = new Revision(333333333L, 0, 1); + + String jsonProperties = "{" + + "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," + + "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" + + "}"; + //prepareDocumentMock(jsonProperties); + + insertJsonDataToNodeDocument(workingDocument, jsonProperties); + + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + + assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); + assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + }*/ + + /*private void insertJsonDataToNodeDocument(NodeDocument document, String jsonProperties) throws Exception { + String json = jsonProperties.replace("'", "\""); + + ObjectMapper mapper = new ObjectMapper(); + // TypeReference helps in specifying the complex type (Map<String, Object>) + Map<String, Object> parsedData = mapper.readValue(json, new TypeReference<Map<String, Object>>() {}); + + // Since the data field is a Map<String, Object>, we need to ensure the inner maps are also correctly typed + for (Map.Entry<String, Object> entry : parsedData.entrySet()) { + if (entry.getValue() instanceof Map) { + Map<?, ?> innerMap = (Map<?, ?>) entry.getValue(); + Map<String, Object> correctedInnerMap = new TreeMap<>(); + for (Map.Entry<?, ?> innerEntry : innerMap.entrySet()) { + correctedInnerMap.put(innerEntry.getKey().toString(), innerEntry.getValue()); + } + parsedData.put(entry.getKey(), correctedInnerMap); + } + } + + document.data = parsedData; // Directly setting the protected field + }*/ + + /*@Test + public void testtest() throws CommitFailedException, NoSuchFieldException, IllegalAccessException { + DocumentStore store = new MemoryDocumentStore(); + DocumentNodeStore ns = new DocumentMK.Builder().setDocumentStore(store).setAsyncDelay(0).getNodeStore(); + + // add properties + for (int i = 0; i < 10; i++) { + NodeBuilder nb = ns.getRoot().builder(); + + nb.child("x").setProperty("p"+i, i); + merge(ns, nb); + } + + NodeDocument nodeDocument = store.find(NODES, "1:/x"); + System.out.println(nodeDocument.getPropertyNames()); + + // Mocked data + Map<String, Object> data = Maps.newHashMap(); + + + NodeDocument document = new NodeDocument(store, System.currentTimeMillis()); + document.data.put("foo", "bar"); + System.out.println(document.getPropertyNames()); + + document.data = Maps.newHashMap(); + + + System.out.println(document.getPropertyNames()); + }*/ + @Test public void testClassifyAndMapRevisionsAndPropertiesWithDeleted() throws IOException { Revision revisionA = new Revision(111111111L, 0, 1); @@ -348,14 +544,14 @@ public class DocumentRevisionCleanupHelperTest { "}"; prepareDocumentMock(jsonProperties); - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision(); + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); assertEquals(Set.of("prop1", "_deleted"), propertiesModifiedByRevision.get(revisionA)); assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster(); + SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1)); assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1)); assertEquals(Set.of(revisionA), revisionsModifyingProperty.get("_deleted").get(1)); @@ -373,14 +569,14 @@ public class DocumentRevisionCleanupHelperTest { "}"; prepareDocumentMock(jsonProperties); - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision(); + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionA)); assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionB)); assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionC)); - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster(); + SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("_deleted").get(1)); } @@ -397,14 +593,14 @@ public class DocumentRevisionCleanupHelperTest { "}"; prepareDocumentMock(jsonProperties); - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision(); + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA)); assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster(); + SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1)); assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1)); } @@ -423,16 +619,16 @@ public class DocumentRevisionCleanupHelperTest { "}"; prepareDocumentMock(jsonProperties); - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); // Modifications done in revisionD should be ignored, as it is not committed - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision(); + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA)); assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); assertNull(propertiesModifiedByRevision.get(revisionD)); - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster(); + SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1)); assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1)); } @@ -451,15 +647,15 @@ public class DocumentRevisionCleanupHelperTest { "}"; prepareDocumentMock(jsonProperties); - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); + NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision(); + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA)); assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionD)); - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster(); + SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); assertEquals(Set.of(revisionA, revisionD), revisionsModifyingProperty.get("prop1").get(1)); assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop1").get(2)); assertEquals(Set.of(revisionC), revisionsModifyingProperty.get("prop1").get(3)); @@ -500,9 +696,10 @@ public class DocumentRevisionCleanupHelperTest { entries.put(property, sortedRevisions); } - Mockito.when(workingDocument.entrySet()).thenReturn(entries.entrySet()); - Mockito.when(workingDocument.get("_deleted")).thenReturn(entries.get("_deleted")); - Mockito.when(workingDocument.get("_revisions")).thenReturn(allRevisions); + //System.out.println("ENTRIES::" + entries); + //System.out.println("DATA::" + data); + + workingDocument.data = entries; } private void prepareCheckpointsMock(String jsonCheckpoints) throws IOException {
