This is an automated email from the ASF dual-hosted git repository. joscorbe pushed a commit to branch old-revisions-cleanup-rebase in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 29d13489ccd9491f9feb11055ce009d2b15e20f4 Author: Jose Cordero <[email protected]> AuthorDate: Wed Feb 7 14:33:21 2024 +0100 Some cleanup and work on RevisionCommand (WIP). --- .../oak/console/commands/RevisionCommand.groovy | 19 +- .../jackrabbit/oak/run/RevisionsCommand.java | 73 +-- .../document/DocumentRevisionCleanupHelper.java | 287 ---------- .../document/NodeDocumentRevisionCleaner.java | 188 +++---- .../DocumentRevisionCleanupHelperTest.java | 620 --------------------- .../document/NodeDocumentRevisionCleanerTest.java | 298 ++-------- 6 files changed, 160 insertions(+), 1325 deletions(-) diff --git a/oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/RevisionCommand.groovy b/oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/RevisionCommand.groovy index 8e055d17a9..81ddc2af8d 100644 --- a/oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/RevisionCommand.groovy +++ b/oak-run/src/main/groovy/org/apache/jackrabbit/oak/console/commands/RevisionCommand.groovy @@ -22,8 +22,8 @@ package org.apache.jackrabbit.oak.console.commands import groovy.transform.CompileStatic import org.apache.jackrabbit.oak.console.ConsoleSession import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore -import org.apache.jackrabbit.oak.plugins.document.DocumentRevisionCleanupHelper import org.apache.jackrabbit.oak.plugins.document.DocumentStore +import org.apache.jackrabbit.oak.plugins.document.NodeDocumentRevisionCleaner import org.apache.jackrabbit.oak.plugins.document.Revision import org.apache.jackrabbit.oak.spi.state.NodeStore import org.codehaus.groovy.tools.shell.CommandSupport @@ -56,7 +56,7 @@ class RevisionCommand extends CommandSupport { private static boolean capResults = true private static boolean showPropertiesUse = false - private DocumentRevisionCleanupHelper cleanupHelper; + private NodeDocumentRevisionCleaner cleanupHelper; RevisionCommand(Groovysh shell) { super(shell, COMMAND_NAME, 'rev') @@ -105,9 +105,9 @@ class RevisionCommand extends CommandSupport { */ void list() { int count = 0 - NavigableMap<Revision, String> allRevisions = cleanupHelper.getAllRevisions(); + SortedMap<Revision, String> allRevisions = cleanupHelper.getAllRevisions(); - for (Map.Entry<Revision, String> revisionEntry : order == ORDER_ASC ? allRevisions.entrySet() : allRevisions.descendingMap().entrySet()) { + for (Map.Entry<Revision, String> revisionEntry : allRevisions.entrySet()) { Revision revision = revisionEntry.getKey() String value = revisionEntry.getValue() io.out.println(revision.toReadableString() + " " + value) @@ -172,7 +172,7 @@ class RevisionCommand extends CommandSupport { void cleanup(int numberToCleanup, int clusterToCleanup) { io.out.println("This will delete the following revisions and the property values permanently:") SortedMap<Integer, TreeSet<Revision>> candidateRevisionsToClean = cleanupHelper.getCandidateRevisionsToClean() - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = cleanupHelper.getPropertiesModifiedByRevision() + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = null;//cleanupHelper.getPropertiesModifiedByRevision() TreeSet<Revision> revisions = candidateRevisionsToClean.get(clusterToCleanup) if (revisions != null) { int count = 0 @@ -223,9 +223,10 @@ class RevisionCommand extends CommandSupport { DocumentStore documentStore = nodeStore.getDocumentStore() DocumentNodeStore documentNodeStore = (DocumentNodeStore) nodeStore - cleanupHelper = new DocumentRevisionCleanupHelper(documentStore, documentNodeStore, session.getWorkingPath()) - cleanupHelper.initializeCleanupProcess() + //cleanupHelper = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, session.get) + + //cleanupHelper.initializeCleanupProcess() SortedMap<Revision, String> allRevisions = cleanupHelper.getAllRevisions() TreeMap<Integer, Integer> revisionsByClusterId = allRevisions.keySet().groupBy { it.clusterId } @@ -233,7 +234,7 @@ class RevisionCommand extends CommandSupport { [clusterId, revisions.size()] } as TreeMap<Integer, Integer> - io.out.println("=== Last Revision by clusterId ===") + /*io.out.println("=== Last Revision by clusterId ===") for (Map.Entry<Integer, Revision> entry : cleanupHelper.getLastRev()) { io.out.printf(" [%d] -> %s%n", entry.key, entry.value.toReadableString()) } @@ -243,7 +244,7 @@ class RevisionCommand extends CommandSupport { for (Map.Entry<Integer, Revision> entry : cleanupHelper.getSweepRev()) { io.out.printf(" [%d] -> %s%n", entry.key, entry.value.toReadableString()) } - io.out.println() + io.out.println()*/ io.out.println("=== Total Revisions by clusterId ===") for (Map.Entry<Integer, Integer> entry : revisionsByClusterId) { diff --git a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java index 7c44cefa65..ac72509b26 100644 --- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java +++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java @@ -37,7 +37,6 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.LoggerContext; @@ -47,13 +46,16 @@ import org.apache.jackrabbit.oak.commons.TimeDurationFormatter; import org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfoDocument; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder; -import org.apache.jackrabbit.oak.plugins.document.DocumentRevisionCleanupHelper; import org.apache.jackrabbit.oak.plugins.document.DocumentStore; +import org.apache.jackrabbit.oak.plugins.document.DocumentStoreException; import org.apache.jackrabbit.oak.plugins.document.FormatVersion; import org.apache.jackrabbit.oak.plugins.document.MissingLastRevSeeker; +import org.apache.jackrabbit.oak.plugins.document.NodeDocument; +import org.apache.jackrabbit.oak.plugins.document.NodeDocumentRevisionCleaner; import org.apache.jackrabbit.oak.plugins.document.Revision; import org.apache.jackrabbit.oak.plugins.document.RevisionContextWrapper; import org.apache.jackrabbit.oak.plugins.document.SweepHelper; +import org.apache.jackrabbit.oak.plugins.document.UpdateOp; import org.apache.jackrabbit.oak.plugins.document.VersionGCSupport; import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCInfo; import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats; @@ -66,7 +68,9 @@ import org.slf4j.LoggerFactory; import static java.lang.Boolean.TRUE; import static java.lang.Integer.getInteger; +import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES; import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreHelper.createVersionGC; import static org.apache.jackrabbit.oak.plugins.document.FormatVersion.versionOf; import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getRootDocument; @@ -496,16 +500,6 @@ public class RevisionsCommand implements Command { System.err.println("path option is required for " + RevisionsOptions.CMD_CLEANUP + " command"); return; } - Integer clusterToCleanup = options.getClusterId(); - if (clusterToCleanup == null) { - System.err.println("clusterId option is required for " + RevisionsOptions.CMD_CLEANUP + " command"); - return; - } - Integer numberToCleanup = options.getNumber(); - if (numberToCleanup == null) { - System.err.println("number option is required for " + RevisionsOptions.CMD_CLEANUP + " command"); - return; - } DocumentNodeStoreBuilder<?> builder = createDocumentMKBuilder(options, closer); if (builder == null) { @@ -517,10 +511,14 @@ public class RevisionsCommand implements Command { builder.setReadOnlyMode(); useMemoryBlobStore(builder); DocumentNodeStore documentNodeStore = builder.build(); - DocumentRevisionCleanupHelper cleanupHelper = new DocumentRevisionCleanupHelper(documentStore, documentNodeStore, path); - cleanupHelper.initializeCleanupProcess(); + String id = org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath(path); + NodeDocument workingDocument = documentStore.find(NODES, id); + NodeDocumentRevisionCleaner revisionCleaner = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, workingDocument); + + final UpdateOp op = new UpdateOp(requireNonNull(id), false); + revisionCleaner.initializeCleanupProcess(op); - SortedMap<Revision, String> allRevisions = cleanupHelper.getAllRevisions(); + SortedMap<Revision, String> allRevisions = revisionCleaner.getAllRevisions(); SortedMap<Integer, Integer> revisionsByClusterId = new TreeMap<>(); for (Revision revision : allRevisions.keySet()) { Integer cid = revision.getClusterId(); @@ -531,15 +529,6 @@ public class RevisionsCommand implements Command { revisionsByClusterId.put(cid, count + 1); } - System.out.println("Last Revision by clusterId"); - for (Map.Entry<Integer, Revision> entry : cleanupHelper.getLastRev().entrySet()) { - System.out.printf(" [%d] -> %s%n", entry.getKey(), entry.getValue().toReadableString()); - } - - System.out.println("Sweep Revision by clusterId"); - for (Map.Entry<Integer, Revision> entry : cleanupHelper.getSweepRev().entrySet()) { - System.out.printf(" [%d] -> %s%n", entry.getKey(), entry.getValue().toReadableString()); - } System.out.println(); int count = 0; @@ -553,15 +542,18 @@ public class RevisionsCommand implements Command { } } - for (Map.Entry<Integer, TreeSet<Revision>> entry : cleanupHelper.getCandidateRevisionsToClean().entrySet()) { + for (Map.Entry<Integer, TreeSet<Revision>> entry : revisionCleaner.getCandidateRevisionsToClean().entrySet()) { Integer cid = entry.getKey(); TreeSet<Revision> revisions = entry.getValue(); - int blocked = cleanupHelper.getBlockedRevisionsToKeep().get(cid) != null ? cleanupHelper.getBlockedRevisionsToKeep().get(cid).size() : 0; + int blocked = revisionCleaner.getBlockedRevisionsToKeep().get(cid) != null ? revisionCleaner.getBlockedRevisionsToKeep().get(cid).size() : 0; System.out.printf("ClusterId [%d] has %d Candidates and %d Blocked%n", cid, revisions.size(), blocked); } - System.out.println("=== Revisions to be cleaned for clusterId " + clusterToCleanup + " ==="); - TreeSet<Revision> revisionsToClean = cleanupHelper.getCandidateRevisionsToClean().get(clusterToCleanup) + System.out.println("=== Operations to be executed ==="); + System.out.println("Total of operations: " + op.getChanges().size()); + System.out.println(op.getChanges()); + + /*TreeSet<Revision> revisionsToClean = revisionCleaner.getCandidateRevisionsToClean().get(clusterToCleanup) .stream().limit(numberToCleanup).collect(Collectors.toCollection(TreeSet::new)); if (revisionsToClean.isEmpty()) { System.out.println("No revisions to clean"); @@ -577,32 +569,17 @@ public class RevisionsCommand implements Command { break; } } - } + }*/ Scanner scanner = new Scanner(System.in); System.out.println("The revisions will be deleted permanently. Are you sure to proceed? [y/N]"); String confirmation = scanner.nextLine().trim().toLowerCase(); if (confirmation.equals("y") || confirmation.equals("yes")) { - // Start the cleanup - count = 0; - for (Revision revision : cleanupHelper.getCandidateRevisionsToClean().get(clusterToCleanup)) { - // Remove the revision from _revisions and all the properties - /*UpdateOp update = new UpdateOp(workingDocument.path.toString(), false) - update.removeMapEntry("_revisions", revision) - for (String property : propertiesModifiedByRevision.get(revision)) { - update.removeMapEntry(property, revision) - } - io.out.println("Executing UpdateOp: " + update) - try { - documentStore.findAndUpdate(NODES, update) - } catch (DocumentStoreException ex) { - io.out.println("Operation failed: " + ex) - }*/ - + try { count++; - if (count >= numberToCleanup) { - break; - } + documentStore.findAndUpdate(NODES, op); + } catch (DocumentStoreException ex) { + System.out.println("Operation failed: " + ex); } System.out.println("-- Executed " + count + " operations --"); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelper.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelper.java deleted file mode 100644 index 0613444efe..0000000000 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelper.java +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.jackrabbit.oak.plugins.document; - -import org.apache.jackrabbit.oak.plugins.document.util.Utils; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.HashMap; -import java.util.Map; -import java.util.NavigableMap; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.TreeSet; - -import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES; -import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.isDeletedEntry; - -public class DocumentRevisionCleanupHelper { - - private final DocumentStore documentStore; - private final DocumentNodeStore documentNodeStore; - private final NodeDocument rootDoc; - private final NodeDocument workingDocument; - - private SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingPropertyByCluster; - private SortedMap<String, TreeSet<Revision>> revisionsModifyingProperty; - private SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision; - private SortedMap<Integer, TreeSet<Revision>> blockedRevisionsToKeep; - private SortedMap<Integer, TreeSet<Revision>> candidateRevisionsToClean; - - /** - * Constructor for DocumentRevisionCleanupHelper. - * @param documentStore The DocumentStore instance. Must be writable to perform cleanup. - * @param documentNodeStore The DocumentNodeStore instance. - * @param path The path of the document to clean up. - */ - public DocumentRevisionCleanupHelper(DocumentStore documentStore, DocumentNodeStore documentNodeStore, String path) { - this.candidateRevisionsToClean = new TreeMap<>(); - this.blockedRevisionsToKeep = new TreeMap<>(); - this.revisionsModifyingPropertyByCluster = new TreeMap<>(); - this.revisionsModifyingProperty = new TreeMap<>(); - this.propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE); - - this.rootDoc = documentStore.find(NODES, Utils.getIdFromPath("/")); - String id = Utils.getIdFromPath(path); - this.workingDocument = documentStore.find(NODES, id); - this.documentStore = documentStore; - this.documentNodeStore = documentNodeStore; - } - - /** - * Performs the full revision cleanup process for the given document for a clusterId. - */ - public void initializeCleanupProcess() { - classifyAndMapRevisionsAndProperties(); - markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS); - markLastRevisionForEachProperty(); - markCheckpointRevisionsToPreserve(); - removeCandidatesInList(blockedRevisionsToKeep); - } - - public int executeCleanupProcess(int numberToCleanup, int clusterToCleanup) { - return -99; - } - - /** - * Step 1: - * This method processes the revisions of the working document, classifying them into two categories: - * candidate revisions that can be cleaned up and used revisions that should be kept. It also creates maps to - * track the relationships between revisions and properties modified by them. - */ - protected void classifyAndMapRevisionsAndProperties() { - candidateRevisionsToClean = new TreeMap<>(); - blockedRevisionsToKeep = new TreeMap<>(); - revisionsModifyingPropertyByCluster = new TreeMap<>(); - revisionsModifyingProperty = new TreeMap<>(); - propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE); - - // The first entry in "_deleted" has to be kept, as is when the document was created - NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.get("_deleted")); - if (deletedRevisions != null && !deletedRevisions.isEmpty()) { - // TODO: This is just a check to ensure the order is the expected - assert(deletedRevisions.descendingMap().lastKey().getTimestamp() <= deletedRevisions.descendingMap().firstKey().getTimestamp()); - - Revision createdRevision = deletedRevisions.descendingMap().lastKey(); - addBlockedRevisionToKeep(createdRevision); - } - - SortedMap<Revision, String> documentRevisions = getAllRevisions(); - for (Map.Entry<Revision, String> revisionEntry : documentRevisions.entrySet()) { - Revision revision = revisionEntry.getKey(); - String revisionValue = revisionEntry.getValue(); - - // Only check committed revisions (ignore branch commits starting with "c-") - if ("c".equals(revisionValue)) { - // Candidate to clean up - addCandidateRevisionToClean(revision); - // Store properties usage - mapPropertiesModifiedByThisRevision(revision); - } - } - } - - /** - * Step 2: - * This method filters out revisions that are newer than a specified time threshold (specified by amount and unit) - * from the candidate revisions to be cleaned up, marking them as used revisions that should be kept. - * @param amount the amount of time - * @param unit the unit of time - */ - protected void markRevisionsNewerThanThresholdToPreserve(long amount, ChronoUnit unit) { - // TODO: Should we add a buffer here? Maybe 1 minute or even 1 hour? - long thresholdToPreserve = Instant.now().minus(amount, unit).toEpochMilli(); - for (TreeSet<Revision> revisionSet : candidateRevisionsToClean.values()) { - for (Revision revision : revisionSet) { - if (revision.getTimestamp() > thresholdToPreserve) { - addBlockedRevisionToKeep(revision); - } - } - } - } - - /** - * Step 3: - * This method processes a set of revisions that modified certain properties and keeps the last revision that - * modified each property. This means, the current status of the node will be preserved, for each clusterId. - */ - protected void markLastRevisionForEachProperty() { - for (SortedMap<Integer, TreeSet<Revision>> revisionsByCluster : revisionsModifyingPropertyByCluster.values()) { - for (TreeSet<Revision> revisions : revisionsByCluster.values()) { - Revision lastRevision = revisions.last(); - addBlockedRevisionToKeep(lastRevision); - } - } - } - - /** - * Step 4: - * Process a set of revisions that modified certain properties and determine which revisions should be - * kept based on the checkpoints. - */ - protected void markCheckpointRevisionsToPreserve() { - SortedMap<Revision, Checkpoints.Info> checkpoints = documentNodeStore.getCheckpoints().getCheckpoints(); - checkpoints.forEach((revision, info) -> { - // For each checkpoint, keep the last revision that modified a property prior to checkpoint - revisionsModifyingProperty.forEach((propertyName, revisionsSet) -> { - // Traverse the revisionVector of the checkpoint and find the last revision that modified the property - info.getCheckpoint().forEach(revisionToFind -> { - // If the exact revision exists, keep it. If not, find the previous one that modified that property - if (revisionsSet.contains(revisionToFind)) { - addBlockedRevisionToKeep(revisionToFind); - } else { - Revision previousRevision = revisionsSet.descendingSet().ceiling(revisionToFind); - if (previousRevision != null) { - addBlockedRevisionToKeep(previousRevision); - } - } - }); - }); - }); - } - - /** - * Step 5: - * Removes for each clusterId the revisions in the Map, that were blocked in the methods above. - */ - protected void removeCandidatesInList(SortedMap<Integer, TreeSet<Revision>> revisions) { - revisions.forEach((key, value) -> { - if (candidateRevisionsToClean.containsKey(key)) { - candidateRevisionsToClean.get(key).removeAll(value); - } - }); - } - - /** - * This method processes a given revision and identify the properties modified by it within the working document. - * It maintains two data structures, propertiesModifiedByRevision and revisionsModifyingProperty, to store - * the properties and their associated revisions. - * @param revision - */ - private void mapPropertiesModifiedByThisRevision(Revision revision) { - for (Map.Entry<String, Object> propertyEntry : workingDocument.entrySet()) { - if (Utils.isPropertyName(propertyEntry.getKey()) || isDeletedEntry(propertyEntry.getKey())) { - Map<Revision, String> valueMap = (Map) propertyEntry.getValue(); - if (valueMap.containsKey(revision)) { - propertiesModifiedByRevision.computeIfAbsent(revision, key -> - new TreeSet<>()).add(propertyEntry.getKey() - ); - - revisionsModifyingPropertyByCluster.computeIfAbsent(propertyEntry.getKey(), key -> - new TreeMap<>() - ).computeIfAbsent(revision.getClusterId(), key -> - new TreeSet<>(StableRevisionComparator.INSTANCE) - ).add(revision); - - revisionsModifyingProperty.computeIfAbsent(propertyEntry.getKey(), key -> - new TreeSet<>(StableRevisionComparator.INSTANCE) - ).add(revision); - } - } - } - } - - /** - * Adds a revision to the list of candidates to delete. - * @param revision - */ - private void addCandidateRevisionToClean(Revision revision) { - candidateRevisionsToClean.computeIfAbsent(revision.getClusterId(), key -> - new TreeSet<>(StableRevisionComparator.INSTANCE) - ).add(revision); - } - - /** - * Adds a revision to the list of revisions to keep. - * @param revision - */ - private void addBlockedRevisionToKeep(Revision revision) { - blockedRevisionsToKeep.computeIfAbsent(revision.getClusterId(), key -> - new TreeSet<>(StableRevisionComparator.INSTANCE) - ).add(revision); - } - - /** - * Returns the LastRev map from the root document. - * @return - */ - public Map<Integer, Revision> getLastRev() { - return rootDoc.getLastRev(); - } - - /** - * Returns the SweepRev map from the root document. - * @return - */ - public Map<Integer, Revision> getSweepRev() { - Map<Integer, Revision> map = new HashMap<>(); - Map<Revision, String> valueMap = (SortedMap<Revision, String>) rootDoc.get("_sweepRev"); - for (Map.Entry<Revision, String> e : valueMap.entrySet()) { - int clusterId = e.getKey().getClusterId(); - Revision rev = Revision.fromString(e.getValue()); - map.put(clusterId, rev); - } - return map; - } - - public NavigableMap<Revision, String> getAllRevisions() { - return (NavigableMap<Revision, String>) workingDocument.get("_revisions"); - } - - public SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> getRevisionsModifyingPropertyByCluster() { - return revisionsModifyingPropertyByCluster; - } - - public SortedMap<String, TreeSet<Revision>> getRevisionsModifyingProperty() { - return revisionsModifyingProperty; - } - - public SortedMap<Revision, TreeSet<String>> getPropertiesModifiedByRevision() { - return propertiesModifiedByRevision; - } - - public SortedMap<Integer, TreeSet<Revision>> getBlockedRevisionsToKeep() { - return blockedRevisionsToKeep; - } - - public SortedMap<Integer, TreeSet<Revision>> getCandidateRevisionsToClean() { - return candidateRevisionsToClean; - } -} diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java index ed28c957ed..b55db4afb6 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleaner.java @@ -22,150 +22,80 @@ import org.apache.jackrabbit.oak.plugins.document.util.Utils; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.HashMap; import java.util.Map; -import java.util.NavigableMap; -import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.TreeSet; -import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES; - public class NodeDocumentRevisionCleaner { private final DocumentStore documentStore; private final DocumentNodeStore documentNodeStore; private final NodeDocument workingDocument; - - protected RevisionClassifierUtility revisionClassifier; - protected RevisionCleanerUtility revisionCleaner; + private final RevisionPropertiesClassifier revisionClassifier; + private final RevisionCleanerUtility revisionCleaner; /** - * Constructor for DocumentRevisionCleanupHelper. + * Constructor for NodeDocumentRevisionCleaner. * @param documentStore The DocumentStore instance. Must be writable to perform cleanup. * @param documentNodeStore The DocumentNodeStore instance. - * @param path The path of the document to clean up. + * @param workingDocument The document to clean up. */ - public NodeDocumentRevisionCleaner(DocumentStore documentStore, DocumentNodeStore documentNodeStore, String path) { - String id = Utils.getIdFromPath(path); - this.workingDocument = documentStore.find(NODES, id); + public NodeDocumentRevisionCleaner(DocumentStore documentStore, DocumentNodeStore documentNodeStore, + NodeDocument workingDocument) { + this.workingDocument = workingDocument; this.documentStore = documentStore; this.documentNodeStore = documentNodeStore; - revisionClassifier = new RevisionClassifierUtility(workingDocument); + revisionClassifier = new RevisionPropertiesClassifier(workingDocument); revisionCleaner = new RevisionCleanerUtility(revisionClassifier); } /** - * Performs the full revision cleanup process for the given document for a clusterId. + * Collects cleanable old revisions for the given document. */ - public void initializeCleanupProcess() { - revisionClassifier.identifyRevisionsToClean(); - revisionCleaner.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS); - revisionCleaner.markLastRevisionForEachProperty(); - revisionCleaner.markCheckpointRevisionsToPreserve(); + public void collectOldRevisions(UpdateOp op) { + revisionClassifier.classifyRevisionsAndProperties(); + revisionCleaner.preserveRevisionsNewerThanThreshold(24, ChronoUnit.HOURS); + revisionCleaner.preserveLastRevisionForEachProperty(); + revisionCleaner.preserveRevisionsReferencedByCheckpoints(); revisionCleaner.removeCandidatesInList(); - } - - public int executeCleanupProcess(int numberToCleanup, int clusterToCleanup) { - return -99; - } - protected void classifyAndMapRevisionsAndProperties() { - //identifyRevisionsToClean(); - //BclassifyAndMapRevisionsAndProperties(); - - revisionClassifier.identifyRevisionsToClean(); - //newClassifyAndMapRevisionsAndProperties(); - } - - /*protected void newClassifyAndMapRevisionsAndProperties() { - candidateRevisionsToClean = new TreeMap<>(); - blockedRevisionsToKeep = new TreeMap<>(); - revisionsModifyingPropertyByCluster = new TreeMap<>(); - revisionsModifyingProperty = new TreeMap<>(); - propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE); - - // The first entry in "_deleted" has to be kept, as is when the document was created - //NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.get("_deleted")); - SortedMap<Revision, String> deletedRevisions = workingDocument.getLocalDeleted(); - if (deletedRevisions != null && !deletedRevisions.isEmpty()) { - Revision createdRevision = deletedRevisions.firstKey(); - addBlockedRevisionToKeep(createdRevision); - } - - SortedMap<Revision, String> documentRevisions = getAllRevisions(); - for (Map.Entry<Revision, String> revisionEntry : documentRevisions.entrySet()) { - Revision revision = revisionEntry.getKey(); - String revisionValue = revisionEntry.getValue(); - - // Only check committed revisions (ignore branch commits starting with "c-") - if ("c".equals(revisionValue)) { - // Candidate to clean up - addCandidateRevisionToClean(revision); - // Store properties usage - mapPropertiesModifiedByThisRevision(revision); - } - } - }*/ - - /** - * Step 1: - * This method processes the revisions of the working document, classifying them into two categories: - * candidate revisions that can be cleaned up and used revisions that should be kept. It also creates maps to - * track the relationships between revisions and properties modified by them. - */ - /*protected void BclassifyAndMapRevisionsAndProperties() { - candidateRevisionsToClean = new TreeMap<>(); - blockedRevisionsToKeep = new TreeMap<>(); - revisionsModifyingPropertyByCluster = new TreeMap<>(); - revisionsModifyingProperty = new TreeMap<>(); - propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE); - - // The first entry in "_deleted" has to be kept, as is when the document was created - //NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.get("_deleted")); - NavigableMap<Revision, String> deletedRevisions = ((NavigableMap<Revision, String>) workingDocument.getLocalDeleted()); - if (deletedRevisions != null && !deletedRevisions.isEmpty()) { - // TODO: This is just a check to ensure the order is the expected - assert(deletedRevisions.descendingMap().lastKey().getTimestamp() <= deletedRevisions.descendingMap().firstKey().getTimestamp()); - - Revision createdRevision = deletedRevisions.descendingMap().lastKey(); - addBlockedRevisionToKeep(createdRevision); - } + /*for (Map.Entry<Integer, TreeSet<Revision>> entry : revisionCleaner.candidateRevisionsToClean.entrySet()) { + for (Revision revision : entry.getValue()) { + System.out.println("Removing revision " + revision); + TreeSet<String> properties = revisionClassifier.propertiesModifiedByRevision.get(revision); + System.out.println("Properties modified by this revision: " + properties); + if (properties != null) { + for (String property : properties) { + op.removeMapEntry(property, revision); + } + } - SortedMap<Revision, String> documentRevisions = getAllRevisions(); - for (Map.Entry<Revision, String> revisionEntry : documentRevisions.entrySet()) { - Revision revision = revisionEntry.getKey(); - String revisionValue = revisionEntry.getValue(); - - // Only check committed revisions (ignore branch commits starting with "c-") - if ("c".equals(revisionValue)) { - // Candidate to clean up - addCandidateRevisionToClean(revision); - // Store properties usage - mapPropertiesModifiedByThisRevision(revision); + op.removeMapEntry("_revisions", revision); } - } - }*/ + }*/ + } - protected class RevisionClassifierUtility { + protected class RevisionPropertiesClassifier { private final NodeDocument workingDocument; - private final SortedMap<Revision, String> documentRevisions; private SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingPropertyByCluster; private SortedMap<String, TreeSet<Revision>> revisionsModifyingProperty; private SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision; - RevisionClassifierUtility(NodeDocument workingDocument) { + private RevisionPropertiesClassifier(NodeDocument workingDocument) { this.workingDocument = workingDocument; - this.documentRevisions = workingDocument.getLocalRevisions(); this.revisionsModifyingPropertyByCluster = new TreeMap<>(); this.revisionsModifyingProperty = new TreeMap<>(); this.propertiesModifiedByRevision = new TreeMap<>(StableRevisionComparator.INSTANCE); } - public void identifyRevisionsToClean() { + /** + * This method processes the revisions of the working document, creating maps to + * track the relationships between revisions and modified properties. + */ + protected void classifyRevisionsAndProperties() { SortedMap<Revision, String> deletedRevisions = workingDocument.getLocalDeleted(); // Always keep the first "_deleted" entry, as is when the document was created if (!deletedRevisions.isEmpty()) { @@ -178,17 +108,17 @@ public class NodeDocumentRevisionCleaner { Revision revision = revisionEntry.getKey(); String revisionValue = revisionEntry.getValue(); - // Only check committed revisions (ignore branch commits starting with "c-") + // Only check committed revisions if (Utils.isCommitted(revisionValue)) { // Candidate to clean up revisionCleaner.addCandidateRevisionToClean(revision); // Store properties usage - mapPropertiesModifiedByThisRevision(revision); + classifyPropertiesModifiedByRevision(revision); } } } - private void mapPropertiesModifiedByThisRevision(Revision revision) { + private void classifyPropertiesModifiedByRevision(Revision revision) { for (Map.Entry<String, Object> propertyEntry : workingDocument.entrySet()) { if (Utils.isPropertyName(propertyEntry.getKey()) || NodeDocument.isDeletedEntry(propertyEntry.getKey())) { Map<Revision, String> valueMap = (Map) propertyEntry.getValue(); @@ -214,17 +144,17 @@ public class NodeDocumentRevisionCleaner { protected class RevisionCleanerUtility { - private SortedMap<Integer, TreeSet<Revision>> blockedRevisionsToKeep; - private SortedMap<Integer, TreeSet<Revision>> candidateRevisionsToClean; - private final RevisionClassifierUtility revisionClassifier; + private final SortedMap<Integer, TreeSet<Revision>> blockedRevisionsToKeep; + private final SortedMap<Integer, TreeSet<Revision>> candidateRevisionsToClean; + private final RevisionPropertiesClassifier revisionClassifier; - protected RevisionCleanerUtility(RevisionClassifierUtility revisionClassifier) { + private RevisionCleanerUtility(RevisionPropertiesClassifier revisionClassifier) { this.revisionClassifier = revisionClassifier; this.candidateRevisionsToClean = new TreeMap<>(); this.blockedRevisionsToKeep = new TreeMap<>(); } - protected void markLastRevisionForEachProperty() { + protected void preserveLastRevisionForEachProperty() { for (SortedMap<Integer, TreeSet<Revision>> revisionsByCluster : revisionClassifier.revisionsModifyingPropertyByCluster.values()) { for (TreeSet<Revision> revisions : revisionsByCluster.values()) { Revision lastRevision = revisions.last(); @@ -233,7 +163,7 @@ public class NodeDocumentRevisionCleaner { } } - protected void markRevisionsNewerThanThresholdToPreserve(long amount, ChronoUnit unit) { + protected void preserveRevisionsNewerThanThreshold(long amount, ChronoUnit unit) { long thresholdToPreserve = Instant.now().minus(amount, unit).toEpochMilli(); for (TreeSet<Revision> revisionSet : candidateRevisionsToClean.values()) { for (Revision revision : revisionSet) { @@ -244,7 +174,7 @@ public class NodeDocumentRevisionCleaner { } } - protected void markCheckpointRevisionsToPreserve() { + protected void preserveRevisionsReferencedByCheckpoints() { SortedMap<Revision, Checkpoints.Info> checkpoints = documentNodeStore.getCheckpoints().getCheckpoints(); checkpoints.forEach((revision, info) -> { // For each checkpoint, keep the last revision that modified a property prior to checkpoint @@ -294,19 +224,19 @@ public class NodeDocumentRevisionCleaner { } } - public NavigableMap<Revision, String> getAllRevisions() { - return (NavigableMap<Revision, String>) workingDocument.getLocalRevisions(); + public SortedMap<Revision, String> getAllRevisions() { + return workingDocument.getLocalRevisions(); } - public SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> getRevisionsModifyingPropertyByCluster() { + protected SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> getRevisionsModifyingPropertyByCluster() { return revisionClassifier.revisionsModifyingPropertyByCluster; } - public SortedMap<String, TreeSet<Revision>> getRevisionsModifyingProperty() { + protected SortedMap<String, TreeSet<Revision>> getRevisionsModifyingProperty() { return revisionClassifier.revisionsModifyingProperty; } - public SortedMap<Revision, TreeSet<String>> getPropertiesModifiedByRevision() { + protected SortedMap<Revision, TreeSet<String>> getPropertiesModifiedByRevision() { return revisionClassifier.propertiesModifiedByRevision; } @@ -317,4 +247,24 @@ public class NodeDocumentRevisionCleaner { public SortedMap<Integer, TreeSet<Revision>> getCandidateRevisionsToClean() { return revisionCleaner.candidateRevisionsToClean; } + + protected void classifyRevisionsAndProperties() { + revisionClassifier.classifyRevisionsAndProperties(); + } + + protected void markLastRevisionForEachProperty() { + revisionCleaner.preserveLastRevisionForEachProperty(); + } + + protected void markRevisionsNewerThanThresholdToPreserve(long amount, ChronoUnit unit) { + revisionCleaner.preserveRevisionsNewerThanThreshold(amount, unit); + } + + protected void markCheckpointRevisionsToPreserve() { + revisionCleaner.preserveRevisionsReferencedByCheckpoints(); + } + + protected void removeCandidatesInList() { + revisionCleaner.removeCandidatesInList(); + } } diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java deleted file mode 100644 index e1f6608c9d..0000000000 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentRevisionCleanupHelperTest.java +++ /dev/null @@ -1,620 +0,0 @@ -package org.apache.jackrabbit.oak.plugins.document; - -import org.apache.jackrabbit.guava.common.collect.Maps; -import org.apache.jackrabbit.oak.api.CommitFailedException; -import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore; -import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES; -import static org.apache.jackrabbit.oak.plugins.document.TestUtils.merge; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -public class DocumentRevisionCleanupHelperTest { - - @Mock - DocumentStore documentStore; - - @Mock - DocumentNodeStore documentNodeStore; - - @Mock - Checkpoints checkpoints; - - @Mock - NodeDocument workingDocument; - - //NodeDocumentRevisionCleaner NodeDocumentRevisionCleaner; - DocumentRevisionCleanupHelper documentRevisionCleanupHelper; - - @Before - public void setUp() { - MockitoAnnotations.initMocks(this); - workingDocument = Mockito.mock(NodeDocument.class); - //workingDocument = new NodeDocument(documentStore); - - Mockito.when(documentStore.find(Mockito.eq(NODES), Mockito.anyString())).thenReturn(workingDocument); - //NodeDocumentRevisionCleaner = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, "/"); - documentRevisionCleanupHelper = new DocumentRevisionCleanupHelper(documentStore, documentNodeStore, "/"); - } - - @Test - public void testMarkCheckpointRevisionsToPreserveOnePropertyOneCluster() throws IOException { - Revision revisionA = Revision.fromString("r100000000-0-1"); - Revision revisionB = Revision.fromString("r105000000-0-1"); - Revision revisionC = Revision.fromString("r110000000-0-1"); - Revision revisionD = Revision.fromString("r115000000-0-1"); - Revision revisionE = Revision.fromString("r120000000-0-1"); - Revision revisionF = Revision.fromString("r125000000-0-1"); - - Revision checkpoint1 = Revision.fromString("r109000000-0-1"); - Revision checkpoint2 = Revision.fromString("r119000000-0-1"); - - String jsonProperties = "{" + - "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3', '" + revisionD + "': 'value4', '" + revisionE + "': 'value5', '" + revisionF + "': 'value6'}, " + - "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c', '" + revisionC + "': 'c', '" + revisionD + "': 'c', '" + revisionE + "': 'c', '" + revisionF + "': 'c'}" + - "}"; - String jsonCheckpoints = "{" + - "'" + checkpoint1 + "': {'expires':'200000000','rv':'r109000000-0-1'}," + - "'" + checkpoint2 + "': {'expires':'200000000','rv':'r119000000-0-1'}" + - "}"; - - - prepareDocumentMock(jsonProperties); - prepareCheckpointsMock(jsonCheckpoints); - - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - documentRevisionCleanupHelper.markCheckpointRevisionsToPreserve(); - documentRevisionCleanupHelper.removeCandidatesInList(documentRevisionCleanupHelper.getBlockedRevisionsToKeep()); - - // The revisions blocked should be: - // - r105000000-0-1 (blocked by checkpoint r109000000-0-1) - // - r115000000-0-1 (blocked by checkpoint r119000000-0-1) - assertEquals(Set.of(revisionB, revisionD), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revisionA, revisionC, revisionE, revisionF), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); - } - - @Test - public void testRecentRevisionsArePreserved() throws IOException { - StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {"); - StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {"); - - Instant currentTime = Instant.now(); - List<Revision> revisions = new ArrayList<>(); - for (int i = 29; i >= 0; i--) { - long timestamp = currentTime.minus(i, ChronoUnit.HOURS).toEpochMilli(); - Revision revision = new Revision(timestamp, 0, 1); - revisions.add(revision); - - jsonPropBuilder.append("'").append(revision).append("': 'value").append(i).append("', "); - jsonRevisionsBuilder.append("'").append(revision).append("': 'c', "); - } - - // 23 hours and 59 minutes ago -> Should be preserved - long timestamp = currentTime.minus(23, ChronoUnit.HOURS).minus(59, ChronoUnit.MINUTES).toEpochMilli(); - Revision revision = new Revision(timestamp, 0, 1); - revisions.add(revision); // revisions[30] - jsonPropBuilder.append("'").append(revision).append("': 'value").append(30).append("', "); - jsonRevisionsBuilder.append("'").append(revision).append("': 'c', "); - - // 24 hours and 1 minute ago -> Should be candidate to clean - timestamp = currentTime.minus(24, ChronoUnit.HOURS).minus(1, ChronoUnit.MINUTES).toEpochMilli(); - revision = new Revision(timestamp, 0, 1); - revisions.add(revision); // revisions[31] - jsonPropBuilder.append("'").append(revision).append("': 'value").append(30).append("'"); - jsonRevisionsBuilder.append("'").append(revision).append("': 'c'"); - - jsonPropBuilder.append("}"); - jsonRevisionsBuilder.append("}"); - StringBuilder jsonBuilder = new StringBuilder("{"); - jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append("}"); - - // revisions.forEach(rev -> System.out.println(rev.toReadableString())); - - prepareDocumentMock(jsonBuilder.toString()); - - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - documentRevisionCleanupHelper.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS); - documentRevisionCleanupHelper.removeCandidatesInList(documentRevisionCleanupHelper.getBlockedRevisionsToKeep()); - - // The candidate revisions should be the 6 oldest ones (current time -29, -28, -27, -26, -25, -24) - // and the one created 24 hours and 1 minute ago - Set<Revision> expectedCandidateRevisions = Stream.concat( - revisions.subList(0, 6).stream(), Stream.of(revisions.get(31)) - ).collect(Collectors.toSet()); - - // Blocked revisions are all the 24 revisions created in the last 24 hours, and the one created 23 hours and 59 minutes ago - Set<Revision> expectedBlockedRevisions = revisions.subList(6, 31).stream().collect(Collectors.toSet()); - - assertEquals(expectedCandidateRevisions, documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); - assertEquals(expectedBlockedRevisions, documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1)); - } - - @Test - public void testInitializeCleanupProcessMultipleClusters() throws IOException { - List<Revision> revs = new ArrayList<>(); - - // Some initial revisions (0-2) - revs.add(Revision.fromString("r100000000-0-1")); - revs.add(Revision.fromString("r100000001-0-2")); - revs.add(Revision.fromString("r100000002-0-3")); - - // Blocked by first checkpoint r109000000 (3-5) - revs.add(Revision.fromString("r106000003-0-1")); - revs.add(Revision.fromString("r107000004-0-2")); - revs.add(Revision.fromString("r108000005-0-3")); - - // Some unblocked revisions in middle (6-10) - revs.add(Revision.fromString("r110000006-0-3")); - revs.add(Revision.fromString("r110000006-1-3")); - revs.add(Revision.fromString("r114000007-0-2")); - revs.add(Revision.fromString("r115000008-0-1")); - revs.add(Revision.fromString("r118000009-0-1")); - - // Blocked by second checkpoint r118000000 (11-13) - revs.add(Revision.fromString("r118000009-1-1")); - revs.add(Revision.fromString("r118000010-0-2")); - revs.add(Revision.fromString("r118000011-0-3")); - - // Some more unblocked revisions (14-19) - revs.add(Revision.fromString("r120000012-0-1")); - revs.add(Revision.fromString("r122000013-0-2")); - revs.add(Revision.fromString("r122000013-1-2")); - revs.add(Revision.fromString("r123000014-0-2")); - revs.add(Revision.fromString("r125000015-0-3")); - revs.add(Revision.fromString("r130000016-0-1")); - - // Last revision (20-22) - revs.add(Revision.fromString("r130000016-1-1")); - revs.add(Revision.fromString("r130000017-0-2")); - revs.add(Revision.fromString("r130000018-0-3")); - - // Checkpoint revisions - Revision checkpoint1 = Revision.fromString("r109000000-0-1"); - Revision checkpoint2 = Revision.fromString("r119000000-0-1"); - - StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {"); - StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {"); - - for (int i = 0; i < revs.size(); i++) { - jsonPropBuilder.append("'").append(revs.get(i)).append("': 'value").append(i).append("'"); - jsonRevisionsBuilder.append("'").append(revs.get(i)).append("': 'c'"); - if (i < revs.size() - 1) { - jsonPropBuilder.append(", "); - jsonRevisionsBuilder.append(", "); - } - } - - jsonPropBuilder.append("}"); - jsonRevisionsBuilder.append("}"); - StringBuilder jsonBuilder = new StringBuilder("{"); - jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append("}"); - - String jsonCheckpoints = "{" + - "'" + checkpoint1 + "': {'expires':'200000000','rv':'r106500003-0-1,r107500004-0-2,r108500005-0-3'}," + - "'" + checkpoint2 + "': {'expires':'200000000','rv':'r118000009-1-1,r118000010-0-2,r118000011-0-3'}" + - "}"; - - prepareDocumentMock(jsonBuilder.toString()); - prepareCheckpointsMock(jsonCheckpoints); - - documentRevisionCleanupHelper.initializeCleanupProcess(); - - // The revisions blocked should be: - // - r106000003-0-3, r118000004-0-2, r108000005-0-3 (referenced by checkpoint 1) - // - r118000009-1-1, r118000010-0-2, r118000011-0-3 (referenced by checkpoint 2) - // - r130000016-1-1, r130000017-0-2, r130000018-0-3 (last revision) - assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(2)); - assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(3)); - - // Rest of revisions are candidates to clean - assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); - assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(2)); - assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(3)); - - assertTrue(Collections.disjoint(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1))); - assertTrue(Collections.disjoint(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(2), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(2))); - assertTrue(Collections.disjoint(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(3), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(3))); - } - - @Test - public void testCheckpointedRevisionFallback() throws IOException { - List<Revision> revs = new ArrayList<>(); - - revs.add(Revision.fromString("r100000000-0-1")); - revs.add(Revision.fromString("r100000001-0-2")); - revs.add(Revision.fromString("r100000002-0-3")); - - revs.add(Revision.fromString("r101100003-0-1")); - revs.add(Revision.fromString("r101200004-0-2")); - revs.add(Revision.fromString("r101300005-0-3")); - - revs.add(Revision.fromString("r102000006-0-1")); - revs.add(Revision.fromString("r102000007-0-3")); - - revs.add(Revision.fromString("r103000008-0-1")); - revs.add(Revision.fromString("r103000009-0-2")); - revs.add(Revision.fromString("r103000010-0-3")); - - // Checkpoint - Revision checkpoint1 = Revision.fromString("r102000000-0-1"); - - StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {"); - StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {"); - - for (int i = 0; i < revs.size(); i++) { - jsonPropBuilder.append("'").append(revs.get(i)).append("': 'value").append(i).append("'"); - jsonRevisionsBuilder.append("'").append(revs.get(i)).append("': 'c'"); - if (i < revs.size() - 1) { - jsonPropBuilder.append(", "); - jsonRevisionsBuilder.append(", "); - } - } - - jsonPropBuilder.append("}"); - jsonRevisionsBuilder.append("}"); - StringBuilder jsonBuilder = new StringBuilder("{"); - jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append("}"); - - String jsonCheckpoints = "{" + - "'" + checkpoint1 + "': {'expires':'200000000','rv':'r102000007-0-1,r102000000-0-2,r102000008-0-3'}" + - "}"; - - prepareDocumentMock(jsonBuilder.toString()); - prepareCheckpointsMock(jsonCheckpoints); - - documentRevisionCleanupHelper.initializeCleanupProcess(); - - // The revisions blocked should be: - // - r103000008-0-1, r103000009-0-2, r103000010-0-3 (last revisions) - // - r102000006-0-1, r101300005-0-3, r102000007-0-3 (referenced by checkpoint for clusters 1, 2 and 3 respectively) - assertEquals(Set.of(revs.get(6), revs.get(8)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revs.get(9)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(2)); - assertEquals(Set.of(revs.get(5), revs.get(7), revs.get(10)), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(3)); - - // Rest of revisions are candidates to clean - assertEquals(Set.of(revs.get(0), revs.get(3)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); - assertEquals(Set.of(revs.get(1), revs.get(4)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(2)); - assertEquals(Set.of(revs.get(2)), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(3)); - } - - @Test - public void testLastRevisionIsBlocked() throws IOException { - Revision revisionA = new Revision(111111111L, 0, 1); - Revision revisionB = new Revision(222222222L, 0, 1); - Revision revisionC = new Revision(333333333L, 0, 1); - - String jsonProperties = "{" + - "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3'}, " + - "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" + - "}"; - prepareDocumentMock(jsonProperties); - - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - documentRevisionCleanupHelper.markLastRevisionForEachProperty(); - - assertFalse(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1).contains(revisionA)); - assertFalse(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1).contains(revisionB)); - assertEquals(Set.of(revisionC), documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revisionA, revisionB, revisionC), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); - } - - @Test - public void testFirstDeletedRevisionIsBlocked() throws Exception { - Revision revisionA = new Revision(111111111L, 0, 1); - Revision revisionB = new Revision(222222222L, 0, 1); - Revision revisionC = new Revision(333333333L, 0, 1); - - String jsonProperties = "{" + - "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," + - "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" + - "}"; - prepareDocumentMock(jsonProperties); - - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - - assertTrue(documentRevisionCleanupHelper.getBlockedRevisionsToKeep().get(1).contains(revisionA)); - assertEquals(Set.of(revisionA, revisionB, revisionC), documentRevisionCleanupHelper.getCandidateRevisionsToClean().get(1)); - } - - /*@Test - public void testFirstDeletedRevisionIsBlocked() throws Exception { - Revision revisionA = new Revision(111111111L, 0, 1); - Revision revisionB = new Revision(222222222L, 0, 1); - Revision revisionC = new Revision(333333333L, 0, 1); - - String jsonProperties = "{" + - "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," + - "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" + - "}"; - //prepareDocumentMock(jsonProperties); - - insertJsonDataToNodeDocument(workingDocument, jsonProperties); - - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); - - assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); - assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); - }*/ - - /*private void insertJsonDataToNodeDocument(NodeDocument document, String jsonProperties) throws Exception { - String json = jsonProperties.replace("'", "\""); - - ObjectMapper mapper = new ObjectMapper(); - // TypeReference helps in specifying the complex type (Map<String, Object>) - Map<String, Object> parsedData = mapper.readValue(json, new TypeReference<Map<String, Object>>() {}); - - // Since the data field is a Map<String, Object>, we need to ensure the inner maps are also correctly typed - for (Map.Entry<String, Object> entry : parsedData.entrySet()) { - if (entry.getValue() instanceof Map) { - Map<?, ?> innerMap = (Map<?, ?>) entry.getValue(); - Map<String, Object> correctedInnerMap = new TreeMap<>(); - for (Map.Entry<?, ?> innerEntry : innerMap.entrySet()) { - correctedInnerMap.put(innerEntry.getKey().toString(), innerEntry.getValue()); - } - parsedData.put(entry.getKey(), correctedInnerMap); - } - } - - document.data = parsedData; // Directly setting the protected field - }*/ - - /*@Test - public void testtest() throws CommitFailedException, NoSuchFieldException, IllegalAccessException { - DocumentStore store = new MemoryDocumentStore(); - DocumentNodeStore ns = new DocumentMK.Builder().setDocumentStore(store).setAsyncDelay(0).getNodeStore(); - - // add properties - for (int i = 0; i < 10; i++) { - NodeBuilder nb = ns.getRoot().builder(); - - nb.child("x").setProperty("p"+i, i); - merge(ns, nb); - } - - NodeDocument nodeDocument = store.find(NODES, "1:/x"); - System.out.println(nodeDocument.getPropertyNames()); - - // Mocked data - Map<String, Object> data = Maps.newHashMap(); - - - NodeDocument document = new NodeDocument(store, System.currentTimeMillis()); - document.data.put("foo", "bar"); - System.out.println(document.getPropertyNames()); - - document.data = Maps.newHashMap(); - - - System.out.println(document.getPropertyNames()); - }*/ - - @Test - public void testClassifyAndMapRevisionsAndPropertiesWithDeleted() throws IOException { - Revision revisionA = new Revision(111111111L, 0, 1); - Revision revisionB = new Revision(222222222L, 0, 1); - Revision revisionC = new Revision(333333333L, 0, 1); - - String jsonProperties = "{" + - "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3'}, " + - "'prop2': {'" + revisionB + "': 'value4'}, " + - "'_deleted': {'" + revisionA + "': 'false'}," + - "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" + - "}"; - prepareDocumentMock(jsonProperties); - - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision(); - assertEquals(Set.of("prop1", "_deleted"), propertiesModifiedByRevision.get(revisionA)); - assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); - assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); - - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster(); - assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1)); - assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1)); - assertEquals(Set.of(revisionA), revisionsModifyingProperty.get("_deleted").get(1)); - } - - @Test - public void testClassifyAndMapRevisionsMultipleDeleted() throws IOException { - Revision revisionA = new Revision(111111111L, 0, 1); - Revision revisionB = new Revision(222222222L, 0, 1); - Revision revisionC = new Revision(333333333L, 0, 1); - - String jsonProperties = "{" + - "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," + - "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" + - "}"; - prepareDocumentMock(jsonProperties); - - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision(); - assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionA)); - assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionB)); - assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionC)); - - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster(); - assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("_deleted").get(1)); - } - - @Test - public void testClassifyAndMapRevisionsAndPropertiesWithoutDeleted() throws IOException { - Revision revisionA = new Revision(111111111L, 0, 1); - Revision revisionB = new Revision(222222222L, 0, 1); - Revision revisionC = new Revision(333333333L, 0, 1); - - String jsonProperties = "{" + - "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3'}, " + - "'prop2': {'" + revisionB + "': 'value4'}, " + - "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" + - "}"; - prepareDocumentMock(jsonProperties); - - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision(); - assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA)); - assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); - assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); - - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster(); - assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1)); - assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1)); - } - - @Test - public void testClassifyAndMapRevisionsAndPropertiesNotCommitted() throws IOException { - Revision revisionA = new Revision(111111111L, 0, 1); - Revision revisionB = new Revision(222222222L, 0, 1); - Revision revisionC = new Revision(333333333L, 0, 1); - Revision revisionD = new Revision(444444444L, 0, 1); - - String jsonProperties = "{" + - "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3', '" + revisionD + "': 'value5'}, " + - "'prop2': {'" + revisionB + "': 'value4', '" + revisionD + "': 'value5'}, " + - "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c', '" + revisionD + "': 'nc'}" + - "}"; - prepareDocumentMock(jsonProperties); - - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - - // Modifications done in revisionD should be ignored, as it is not committed - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision(); - assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA)); - assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); - assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); - assertNull(propertiesModifiedByRevision.get(revisionD)); - - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster(); - assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1)); - assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1)); - } - - @Test - public void testClassifyAndMapRevisionsAndPropertiesDifferentClusters() throws IOException { - Revision revisionA = new Revision(111111111L, 0, 1); - Revision revisionB = new Revision(222222222L, 0, 2); - Revision revisionC = new Revision(333333333L, 0, 3); - Revision revisionD = new Revision(444444444L, 0, 1); - - String jsonProperties = "{" + - "'prop1': {'" + revisionA + "': 'value1', '" + revisionB + "': 'value2', '" + revisionC + "': 'value3', '" + revisionD + "': 'value5'}, " + - "'prop2': {'" + revisionB + "': 'value4', '" + revisionD + "': 'value5'}, " + - "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c', '" + revisionD + "': 'c'}" + - "}"; - prepareDocumentMock(jsonProperties); - - documentRevisionCleanupHelper.classifyAndMapRevisionsAndProperties(); - - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = documentRevisionCleanupHelper.getPropertiesModifiedByRevision(); - assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA)); - assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); - assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); - assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionD)); - - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = documentRevisionCleanupHelper.getRevisionsModifyingPropertyByCluster(); - assertEquals(Set.of(revisionA, revisionD), revisionsModifyingProperty.get("prop1").get(1)); - assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop1").get(2)); - assertEquals(Set.of(revisionC), revisionsModifyingProperty.get("prop1").get(3)); - assertEquals(Set.of(revisionD), revisionsModifyingProperty.get("prop2").get(1)); - assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(2)); - assertNull(revisionsModifyingProperty.get("prop2").get(3)); - } - - private void prepareDocumentMock(String jsonProperties) throws IOException { - String json = jsonProperties.replace("'", "\""); - - ObjectMapper objectMapper = new ObjectMapper(); - Map<String, Map<String, String>> data = objectMapper.readValue(json, new TypeReference<>() {}); - - SortedMap<String, Object> entries = new TreeMap<>(); - SortedMap<Revision, String> allRevisions = new TreeMap<>(StableRevisionComparator.INSTANCE); - for (Map.Entry<String, Map<String, String>> entry : data.entrySet()) { - String property = entry.getKey(); - Map<String, String> revisions = entry.getValue(); - - SortedMap<Revision, String> sortedRevisions = new TreeMap<>(StableRevisionComparator.INSTANCE); - for (Map.Entry<String, String> revisionEntry : revisions.entrySet()) { - String revisionStr = revisionEntry.getKey(); - String value = revisionEntry.getValue(); - - String[] parts = revisionStr.split("-"); - // The timestamp part of the revision string (first part) is parsed as hexadecimal (radix 16) - long timestamp = Long.parseLong(parts[0].substring(1), 16); - int counter = Integer.parseInt(parts[1]); - int clusterId = Integer.parseInt(parts[2]); - - Revision revision = new Revision(timestamp, counter, clusterId); - sortedRevisions.put(revision, value); - - // Add all revisions to the "_revisions" list - allRevisions.put(revision, value); - } - entries.put(property, sortedRevisions); - } - - Mockito.when(workingDocument.entrySet()).thenReturn(entries.entrySet()); - Mockito.when(workingDocument.get("_deleted")).thenReturn(entries.get("_deleted")); - Mockito.when(workingDocument.get("_revisions")).thenReturn(allRevisions); - } - - private void prepareCheckpointsMock(String jsonCheckpoints) throws IOException { - String json = jsonCheckpoints.replace("'", "\""); - - ObjectMapper objectMapper = new ObjectMapper(); - Map<String, Map<String, Object>> data = objectMapper.readValue(json, new TypeReference<>() {}); - - SortedMap<Revision, Checkpoints.Info> checkpoints = new TreeMap<>(StableRevisionComparator.REVERSE); - for (Map.Entry<String, Map<String, Object>> entry : data.entrySet()) { - String checkpointStr = entry.getKey(); - Map<String, Object> checkpointData = entry.getValue(); - - String[] parts = checkpointStr.split("-"); - long timestamp = Long.parseLong(parts[0].substring(1), 16); - int counter = Integer.parseInt(parts[1]); - int clusterId = Integer.parseInt(parts[2]); - - Revision checkpoint = new Revision(timestamp, counter, clusterId); - String checkpointDataJson = objectMapper.writeValueAsString(checkpointData); - - Checkpoints.Info info = Checkpoints.Info.fromString(checkpointDataJson); - checkpoints.put(checkpoint, info); - } - - Mockito.when(documentNodeStore.getCheckpoints()).thenReturn(this.checkpoints); - Mockito.when(documentNodeStore.getCheckpoints().getCheckpoints()).thenReturn(checkpoints); - - /*Document cdoc = SETTINGS.newDocument(documentStore); - Mockito.when(documentNodeStore.getCheckpoints()).thenReturn(this.checkpoints); - Mockito.when(documentStore.find(Collection.SETTINGS, "checkpoint", 0)).thenReturn(cdoc);*/ - } -} \ No newline at end of file diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java index 4cd9731c21..ab0880ea37 100644 --- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java +++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/NodeDocumentRevisionCleanerTest.java @@ -2,7 +2,6 @@ package org.apache.jackrabbit.oak.plugins.document; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore; import org.junit.Before; import org.junit.Test; import org.mockito.Mock; @@ -40,25 +39,18 @@ public class NodeDocumentRevisionCleanerTest { @Mock Checkpoints checkpoints; - //@Mock NodeDocument workingDocument; - NodeDocumentRevisionCleaner NodeDocumentRevisionCleaner; + NodeDocumentRevisionCleaner nodeDocumentRevisionCleaner; @Before public void setUp() { MockitoAnnotations.initMocks(this); - //workingDocument = Mockito.mock(NodeDocument.class); - - /*documentStore = new MemoryDocumentStore(); - documentNodeStore = new DocumentMK.Builder() - .setDocumentStore(documentStore).setAsyncDelay(0).setClusterId(1).build();*/ - workingDocument = new NodeDocument(documentStore); Mockito.when(documentStore.find(Mockito.eq(NODES), Mockito.anyString())).thenReturn(workingDocument); Mockito.when(documentNodeStore.getBranches()).thenReturn(new UnmergedBranches()); - NodeDocumentRevisionCleaner = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, "/"); + nodeDocumentRevisionCleaner = new NodeDocumentRevisionCleaner(documentStore, documentNodeStore, workingDocument); } @Test @@ -86,15 +78,15 @@ public class NodeDocumentRevisionCleanerTest { prepareDocumentMock(jsonProperties); prepareCheckpointsMock(jsonCheckpoints); - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); - NodeDocumentRevisionCleaner.revisionCleaner.markCheckpointRevisionsToPreserve(); - NodeDocumentRevisionCleaner.revisionCleaner.removeCandidatesInList(); + nodeDocumentRevisionCleaner.classifyRevisionsAndProperties(); + nodeDocumentRevisionCleaner.markCheckpointRevisionsToPreserve(); + nodeDocumentRevisionCleaner.removeCandidatesInList(); // The revisions blocked should be: // - r105000000-0-1 (blocked by checkpoint r109000000-0-1) // - r115000000-0-1 (blocked by checkpoint r119000000-0-1) - assertEquals(Set.of(revisionB, revisionD), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revisionA, revisionC, revisionE, revisionF), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertEquals(Set.of(revisionB, revisionD), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); + assertEquals(Set.of(revisionA, revisionC, revisionE, revisionF), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); } @Test @@ -136,9 +128,9 @@ public class NodeDocumentRevisionCleanerTest { prepareDocumentMock(jsonBuilder.toString()); - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); - NodeDocumentRevisionCleaner.revisionCleaner.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS); - NodeDocumentRevisionCleaner.revisionCleaner.removeCandidatesInList(); + nodeDocumentRevisionCleaner.classifyRevisionsAndProperties(); + nodeDocumentRevisionCleaner.markRevisionsNewerThanThresholdToPreserve(24, ChronoUnit.HOURS); + nodeDocumentRevisionCleaner.removeCandidatesInList(); // The candidate revisions should be the 6 oldest ones (current time -29, -28, -27, -26, -25, -24) // and the one created 24 hours and 1 minute ago @@ -149,8 +141,8 @@ public class NodeDocumentRevisionCleanerTest { // Blocked revisions are all the 24 revisions created in the last 24 hours, and the one created 23 hours and 59 minutes ago Set<Revision> expectedBlockedRevisions = revisions.subList(6, 31).stream().collect(Collectors.toSet()); - assertEquals(expectedCandidateRevisions, NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); - assertEquals(expectedBlockedRevisions, NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); + assertEquals(expectedCandidateRevisions, nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertEquals(expectedBlockedRevisions, nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); } @Test @@ -221,24 +213,24 @@ public class NodeDocumentRevisionCleanerTest { prepareDocumentMock(jsonBuilder.toString()); prepareCheckpointsMock(jsonCheckpoints); - NodeDocumentRevisionCleaner.initializeCleanupProcess(); + nodeDocumentRevisionCleaner.collectOldRevisions(null); // The revisions blocked should be: // - r106000003-0-3, r118000004-0-2, r108000005-0-3 (referenced by checkpoint 1) // - r118000009-1-1, r118000010-0-2, r118000011-0-3 (referenced by checkpoint 2) // - r130000016-1-1, r130000017-0-2, r130000018-0-3 (last revision) - assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2)); - assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3)); + assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); + assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2)); + assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3)); // Rest of revisions are candidates to clean - assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); - assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)); - assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)); + assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)); + assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)); - assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1))); - assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2))); - assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3))); + assertTrue(Collections.disjoint(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1))); + assertTrue(Collections.disjoint(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2))); + assertTrue(Collections.disjoint(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3))); } @Test @@ -287,120 +279,21 @@ public class NodeDocumentRevisionCleanerTest { prepareDocumentMock(jsonBuilder.toString()); prepareCheckpointsMock(jsonCheckpoints); - NodeDocumentRevisionCleaner.initializeCleanupProcess(); + nodeDocumentRevisionCleaner.collectOldRevisions(null); // The revisions blocked should be: // - r103000008-0-1, r103000009-0-2, r103000010-0-3 (last revisions) // - r102000006-0-1, r101300005-0-3, r102000007-0-3 (referenced by checkpoint for clusters 1, 2 and 3 respectively) - assertEquals(Set.of(revs.get(6), revs.get(8)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revs.get(9)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2)); - assertEquals(Set.of(revs.get(5), revs.get(7), revs.get(10)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3)); + assertEquals(Set.of(revs.get(6), revs.get(8)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); + assertEquals(Set.of(revs.get(9)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2)); + assertEquals(Set.of(revs.get(5), revs.get(7), revs.get(10)), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3)); // Rest of revisions are candidates to clean - assertEquals(Set.of(revs.get(0), revs.get(3)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); - assertEquals(Set.of(revs.get(1), revs.get(4)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)); - assertEquals(Set.of(revs.get(2)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)); + assertEquals(Set.of(revs.get(0), revs.get(3)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertEquals(Set.of(revs.get(1), revs.get(4)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)); + assertEquals(Set.of(revs.get(2)), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)); } - // New test copy - /*@Test - public void testInitializeCleanupProcessMultipleClustersNew() throws IOException { - List<Revision> revs = new ArrayList<>(); - - // Some initial revisions (0-2) - revs.add(Revision.fromString("r100000000-0-1")); - revs.add(Revision.fromString("r100000001-0-2")); - revs.add(Revision.fromString("r100000002-0-3")); - - // Blocked by first checkpoint r109000000 (3-5) - revs.add(Revision.fromString("r106000003-0-1")); - revs.add(Revision.fromString("r107000004-0-2")); - revs.add(Revision.fromString("r108000005-0-3")); - - // Some unblocked revisions in middle (6-10) - revs.add(Revision.fromString("r110000006-0-3")); - revs.add(Revision.fromString("r110000006-1-3")); - revs.add(Revision.fromString("r114000007-0-2")); - revs.add(Revision.fromString("r115000008-0-1")); - revs.add(Revision.fromString("r118000009-0-1")); - - // Blocked by second checkpoint r118000000 (11-13) - revs.add(Revision.fromString("r118000009-1-1")); - revs.add(Revision.fromString("r118000010-0-2")); - revs.add(Revision.fromString("r118000011-0-3")); - - // Some more unblocked revisions (14-19) - revs.add(Revision.fromString("r120000012-0-1")); - revs.add(Revision.fromString("r122000013-0-2")); - revs.add(Revision.fromString("r122000013-1-2")); - revs.add(Revision.fromString("r123000014-0-2")); - revs.add(Revision.fromString("r125000015-0-3")); - revs.add(Revision.fromString("r130000016-0-1")); - - // Last revision (20-22) - revs.add(Revision.fromString("r130000016-1-1")); - revs.add(Revision.fromString("r130000017-0-2")); - revs.add(Revision.fromString("r130000018-0-3")); - - // Checkpoint revisions - Revision checkpoint1 = Revision.fromString("r109000000-0-1"); - Revision checkpoint2 = Revision.fromString("r119000000-0-1"); - - StringBuilder jsonPropBuilder = new StringBuilder("'prop1': {"); - StringBuilder jsonRevisionsBuilder = new StringBuilder("'_revisions': {"); - - for (int i = 0; i < revs.size(); i++) { - jsonPropBuilder.append("'").append(revs.get(i)).append("': 'value").append(i).append("'"); - jsonRevisionsBuilder.append("'").append(revs.get(i)).append("': 'c'"); - if (i < revs.size() - 1) { - jsonPropBuilder.append(", "); - jsonRevisionsBuilder.append(", "); - } - } - - jsonPropBuilder.append("}"); - jsonRevisionsBuilder.append("}"); - StringBuilder jsonBuilder = new StringBuilder("{"); - jsonBuilder.append(jsonPropBuilder).append(", ").append(jsonRevisionsBuilder).append(", '_deleted': {'r100000000-0-1': 'false'}").append("}"); - - String jsonCheckpoints = "{" + - "'" + checkpoint1 + "': {'expires':'200000000','rv':'r106500003-0-1,r107500004-0-2,r108500005-0-3'}," + - "'" + checkpoint2 + "': {'expires':'200000000','rv':'r118000009-1-1,r118000010-0-2,r118000011-0-3'}" + - "}"; - - prepareDocumentMock(jsonBuilder.toString()); - prepareCheckpointsMock(jsonCheckpoints); - - NodeDocumentRevisionCleaner.initializeCleanupProcess(); - - RevisionVector readRevision1 = documentNodeStore.getCheckpoints().getCheckpoints().get(checkpoint1).getCheckpoint(); - RevisionVector readRevision2 = documentNodeStore.getCheckpoints().getCheckpoints().get(checkpoint2).getCheckpoint(); - - //workingDocument.getLiveRevision(); - DocumentNodeState state1 = workingDocument.getNodeAtRevision(documentNodeStore, readRevision1,null); - DocumentNodeState state2 = workingDocument.getNodeAtRevision(documentNodeStore, readRevision2,null); - - System.out.println(state1); - System.out.println(state2); - - // The revisions blocked should be: - // - r106000003-0-3, r118000004-0-2, r108000005-0-3 (referenced by checkpoint 1) - // - r118000009-1-1, r118000010-0-2, r118000011-0-3 (referenced by checkpoint 2) - // - r130000016-1-1, r130000017-0-2, r130000018-0-3 (last revision) - /*assertEquals(Set.of(revs.get(3), revs.get(11), revs.get(20)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revs.get(4), revs.get(12), revs.get(21)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2)); - assertEquals(Set.of(revs.get(5), revs.get(13), revs.get(22)), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3)); - - // Rest of revisions are candidates to clean - assertEquals(Set.of(revs.get(0), revs.get(9), revs.get(10), revs.get(14), revs.get(19)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); - assertEquals(Set.of(revs.get(1), revs.get(8), revs.get(15), revs.get(16), revs.get(17)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2)); - assertEquals(Set.of(revs.get(2), revs.get(6), revs.get(7), revs.get(18)), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)); - - assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1))); - assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(2), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(2))); - assertTrue(Collections.disjoint(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(3), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(3)));*/ - //} - @Test public void testLastRevisionIsBlocked() throws IOException { Revision revisionA = new Revision(111111111L, 0, 1); @@ -413,13 +306,13 @@ public class NodeDocumentRevisionCleanerTest { "}"; prepareDocumentMock(jsonProperties); - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); - NodeDocumentRevisionCleaner.revisionCleaner.markLastRevisionForEachProperty(); + nodeDocumentRevisionCleaner.classifyRevisionsAndProperties(); + nodeDocumentRevisionCleaner.markLastRevisionForEachProperty(); - assertFalse(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); - assertFalse(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionB)); - assertEquals(Set.of(revisionC), NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); - assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertFalse(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); + assertFalse(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionB)); + assertEquals(Set.of(revisionC), nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1)); + assertEquals(Set.of(revisionA, revisionB, revisionC), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); } @Test @@ -434,10 +327,10 @@ public class NodeDocumentRevisionCleanerTest { "}"; prepareDocumentMock(jsonProperties); - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + nodeDocumentRevisionCleaner.classifyRevisionsAndProperties(); - assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); - assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertTrue(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); + assertEquals(Set.of(revisionA, revisionB, revisionC), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); } @Test @@ -452,84 +345,12 @@ public class NodeDocumentRevisionCleanerTest { "}"; prepareDocumentMock(jsonProperties); - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + nodeDocumentRevisionCleaner.classifyRevisionsAndProperties(); - assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); - assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); + assertTrue(nodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); + assertEquals(Set.of(revisionA, revisionB, revisionC), nodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); } - /*@Test - public void testFirstDeletedRevisionIsBlocked() throws Exception { - Revision revisionA = new Revision(111111111L, 0, 1); - Revision revisionB = new Revision(222222222L, 0, 1); - Revision revisionC = new Revision(333333333L, 0, 1); - - String jsonProperties = "{" + - "'_deleted': {'" + revisionA + "': 'false', '" + revisionB + "': 'true', '" + revisionC + "': 'false'}," + - "'_revisions': {'" + revisionA + "': 'c', '" + revisionB + "': 'c', '" + revisionC + "': 'c'}" + - "}"; - //prepareDocumentMock(jsonProperties); - - insertJsonDataToNodeDocument(workingDocument, jsonProperties); - - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); - - assertTrue(NodeDocumentRevisionCleaner.getBlockedRevisionsToKeep().get(1).contains(revisionA)); - assertEquals(Set.of(revisionA, revisionB, revisionC), NodeDocumentRevisionCleaner.getCandidateRevisionsToClean().get(1)); - }*/ - - /*private void insertJsonDataToNodeDocument(NodeDocument document, String jsonProperties) throws Exception { - String json = jsonProperties.replace("'", "\""); - - ObjectMapper mapper = new ObjectMapper(); - // TypeReference helps in specifying the complex type (Map<String, Object>) - Map<String, Object> parsedData = mapper.readValue(json, new TypeReference<Map<String, Object>>() {}); - - // Since the data field is a Map<String, Object>, we need to ensure the inner maps are also correctly typed - for (Map.Entry<String, Object> entry : parsedData.entrySet()) { - if (entry.getValue() instanceof Map) { - Map<?, ?> innerMap = (Map<?, ?>) entry.getValue(); - Map<String, Object> correctedInnerMap = new TreeMap<>(); - for (Map.Entry<?, ?> innerEntry : innerMap.entrySet()) { - correctedInnerMap.put(innerEntry.getKey().toString(), innerEntry.getValue()); - } - parsedData.put(entry.getKey(), correctedInnerMap); - } - } - - document.data = parsedData; // Directly setting the protected field - }*/ - - /*@Test - public void testtest() throws CommitFailedException, NoSuchFieldException, IllegalAccessException { - DocumentStore store = new MemoryDocumentStore(); - DocumentNodeStore ns = new DocumentMK.Builder().setDocumentStore(store).setAsyncDelay(0).getNodeStore(); - - // add properties - for (int i = 0; i < 10; i++) { - NodeBuilder nb = ns.getRoot().builder(); - - nb.child("x").setProperty("p"+i, i); - merge(ns, nb); - } - - NodeDocument nodeDocument = store.find(NODES, "1:/x"); - System.out.println(nodeDocument.getPropertyNames()); - - // Mocked data - Map<String, Object> data = Maps.newHashMap(); - - - NodeDocument document = new NodeDocument(store, System.currentTimeMillis()); - document.data.put("foo", "bar"); - System.out.println(document.getPropertyNames()); - - document.data = Maps.newHashMap(); - - - System.out.println(document.getPropertyNames()); - }*/ - @Test public void testClassifyAndMapRevisionsAndPropertiesWithDeleted() throws IOException { Revision revisionA = new Revision(111111111L, 0, 1); @@ -544,14 +365,14 @@ public class NodeDocumentRevisionCleanerTest { "}"; prepareDocumentMock(jsonProperties); - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + nodeDocumentRevisionCleaner.classifyRevisionsAndProperties(); - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); assertEquals(Set.of("prop1", "_deleted"), propertiesModifiedByRevision.get(revisionA)); assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); + SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1)); assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1)); assertEquals(Set.of(revisionA), revisionsModifyingProperty.get("_deleted").get(1)); @@ -569,14 +390,14 @@ public class NodeDocumentRevisionCleanerTest { "}"; prepareDocumentMock(jsonProperties); - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + nodeDocumentRevisionCleaner.classifyRevisionsAndProperties(); - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionA)); assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionB)); assertEquals(Set.of("_deleted"), propertiesModifiedByRevision.get(revisionC)); - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); + SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("_deleted").get(1)); } @@ -593,14 +414,14 @@ public class NodeDocumentRevisionCleanerTest { "}"; prepareDocumentMock(jsonProperties); - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + nodeDocumentRevisionCleaner.classifyRevisionsAndProperties(); - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA)); assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); + SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1)); assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1)); } @@ -619,16 +440,16 @@ public class NodeDocumentRevisionCleanerTest { "}"; prepareDocumentMock(jsonProperties); - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + nodeDocumentRevisionCleaner.classifyRevisionsAndProperties(); // Modifications done in revisionD should be ignored, as it is not committed - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA)); assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); assertNull(propertiesModifiedByRevision.get(revisionD)); - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); + SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); assertEquals(Set.of(revisionA, revisionB, revisionC), revisionsModifyingProperty.get("prop1").get(1)); assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop2").get(1)); } @@ -647,15 +468,15 @@ public class NodeDocumentRevisionCleanerTest { "}"; prepareDocumentMock(jsonProperties); - NodeDocumentRevisionCleaner.classifyAndMapRevisionsAndProperties(); + nodeDocumentRevisionCleaner.classifyRevisionsAndProperties(); - SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = NodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); + SortedMap<Revision, TreeSet<String>> propertiesModifiedByRevision = nodeDocumentRevisionCleaner.getPropertiesModifiedByRevision(); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionA)); assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionB)); assertEquals(Set.of("prop1"), propertiesModifiedByRevision.get(revisionC)); assertEquals(Set.of("prop1", "prop2"), propertiesModifiedByRevision.get(revisionD)); - SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = NodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); + SortedMap<String, SortedMap<Integer, TreeSet<Revision>>> revisionsModifyingProperty = nodeDocumentRevisionCleaner.getRevisionsModifyingPropertyByCluster(); assertEquals(Set.of(revisionA, revisionD), revisionsModifyingProperty.get("prop1").get(1)); assertEquals(Set.of(revisionB), revisionsModifyingProperty.get("prop1").get(2)); assertEquals(Set.of(revisionC), revisionsModifyingProperty.get("prop1").get(3)); @@ -696,9 +517,6 @@ public class NodeDocumentRevisionCleanerTest { entries.put(property, sortedRevisions); } - //System.out.println("ENTRIES::" + entries); - //System.out.println("DATA::" + data); - workingDocument.data = entries; } @@ -727,9 +545,5 @@ public class NodeDocumentRevisionCleanerTest { Mockito.when(documentNodeStore.getCheckpoints()).thenReturn(this.checkpoints); Mockito.when(documentNodeStore.getCheckpoints().getCheckpoints()).thenReturn(checkpoints); - - /*Document cdoc = SETTINGS.newDocument(documentStore); - Mockito.when(documentNodeStore.getCheckpoints()).thenReturn(this.checkpoints); - Mockito.when(documentStore.find(Collection.SETTINGS, "checkpoint", 0)).thenReturn(cdoc);*/ } } \ No newline at end of file
