This is an automated email from the ASF dual-hosted git repository. reschke pushed a commit to branch OAK-10657 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 1e0613436b6e8925fb8b97e7ea75171dddb67812 Author: Julian Reschke <[email protected]> AuthorDate: Wed Feb 21 15:54:03 2024 +0100 OAK-10657: test one-time cleanup for _childOrder property in MemoryDocumentStore --- .../apache/jackrabbit/oak/jcr/ManyChildrenIT.java | 6 +-- .../jackrabbit/oak/plugins/document/UpdateOp.java | 2 +- .../document/memory/MemoryDocumentStore.java | 22 +++++++++- .../oak/plugins/document/util/Utils.java | 51 ++++++++++++++++++++++ 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/ManyChildrenIT.java b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/ManyChildrenIT.java index eeac7c5cc8..dc56586904 100644 --- a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/ManyChildrenIT.java +++ b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/ManyChildrenIT.java @@ -82,7 +82,7 @@ public class ManyChildrenIT extends AbstractRepositoryTest { } @Test - @Ignore //OAK-10646 + // @Ignore //OAK-10646 public void orderableAddManyChildrenWithSave() throws Exception { int childCount = 1000; StringBuilder prefix = new StringBuilder(""); @@ -98,7 +98,7 @@ public class ManyChildrenIT extends AbstractRepositoryTest { } @Test - @Ignore //OAK-10646 + // @Ignore //OAK-10646 public void moveOrderableWithManyChildren() throws Exception { int childCount = 1000; int moveCount = 1; @@ -121,7 +121,7 @@ public class ManyChildrenIT extends AbstractRepositoryTest { } @Test - @Ignore //OAK-10646 + // @Ignore //OAK-10646 public void copyOrderableWithManyChildren() throws Exception { int childCount = 1000; int copyCount = 1; diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UpdateOp.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UpdateOp.java index c3c2e0e0bc..4cd4df63f4 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UpdateOp.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/UpdateOp.java @@ -156,7 +156,7 @@ public final class UpdateOp { * @param revision the revision * @param value the value */ - void setMapEntry(@NotNull String property, @NotNull Revision revision, String value) { + public void setMapEntry(@NotNull String property, @NotNull Revision revision, String value) { Operation op = new Operation(Operation.Type.SET_MAP_ENTRY, value); changes.put(new Key(property, checkNotNull(revision)), op); } diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java index 90af8c10af..2931087815 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/memory/MemoryDocumentStore.java @@ -51,6 +51,8 @@ import org.apache.jackrabbit.oak.plugins.document.cache.CacheInvalidationStats; import org.apache.jackrabbit.oak.plugins.document.util.Utils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.mongodb.ReadPreference; import com.mongodb.WriteConcern; @@ -101,6 +103,8 @@ public class MemoryDocumentStore implements DocumentStore { private static final long SIZE_LIMIT = SystemPropertySupplier.create("memoryds.size.limit", -1).get(); + private static final Logger LOG = LoggerFactory.getLogger(MemoryDocumentStore.class); + public MemoryDocumentStore() { this(false); } @@ -341,7 +345,18 @@ public class MemoryDocumentStore implements DocumentStore { // update the document UpdateUtils.applyChanges(doc, update); maintainModCount(doc); - checkSize(doc); + try { + checkSize(doc); + } catch (DocumentStoreException ex) { + UpdateOp shrink = Utils.getShrinkOp(doc, ":childOrder", "(x)"); + // try cleanup and then retry once + long before = doc.getMemory(); + UpdateUtils.applyChanges(doc, shrink); + long after = doc.getMemory(); + LOG.warn("Doc size was exceeded for {}: {} bytes. Applied shrink ops: {}. New size: {}. Doing one retry.", + doc.getId(), before, shrink, after); + checkSize(doc); + } doc.seal(); map.put(update.getId(), doc); return oldDoc; @@ -474,7 +489,10 @@ public class MemoryDocumentStore implements DocumentStore { return 0; } - private void checkSize(Document doc) { + /** + * aborts the operation if a size limit is configured and exceeded + */ + private void checkSize(Document doc) throws DocumentStoreException { if (SIZE_LIMIT >= 0) { int size = doc.getMemory(); if (size >= SIZE_LIMIT) { diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java index 0920eb056b..3ff21cb49e 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/util/Utils.java @@ -26,6 +26,7 @@ import java.sql.Timestamp; import java.time.Instant; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.Iterator; @@ -46,6 +47,7 @@ import org.apache.jackrabbit.oak.commons.StringUtils; import org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfo; import org.apache.jackrabbit.oak.plugins.document.ClusterNodeInfoDocument; import org.apache.jackrabbit.oak.plugins.document.Collection; +import org.apache.jackrabbit.oak.plugins.document.Document; import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder; import org.apache.jackrabbit.oak.plugins.document.DocumentStore; import org.apache.jackrabbit.oak.plugins.document.DocumentStoreException; @@ -54,6 +56,7 @@ import org.apache.jackrabbit.oak.plugins.document.Path; import org.apache.jackrabbit.oak.plugins.document.Revision; import org.apache.jackrabbit.oak.plugins.document.RevisionVector; import org.apache.jackrabbit.oak.plugins.document.StableRevisionComparator; +import org.apache.jackrabbit.oak.plugins.document.UpdateOp; import org.apache.jackrabbit.oak.spi.toggle.Feature; import org.apache.jackrabbit.oak.stats.Clock; import org.jetbrains.annotations.NotNull; @@ -274,6 +277,54 @@ public class Utils { } } + /** + * Produce an {@link UpdateOp} suitable for shrinking branch revision entries for given property in {@link Document}, {@code null} otherwise. + */ + public static @Nullable UpdateOp getShrinkOp(Document doc, String propertyName, String replacementValue) { + Object t_bc = doc.get("_bc"); + Object t_property = doc.get(propertyName); + if (t_bc instanceof Map && t_property instanceof Map) { + @SuppressWarnings("unchecked") + Map<Revision, String> _bc = (Map<Revision, String>)t_bc; + @SuppressWarnings("unchecked") + Map<Revision, String> pMap = (Map<Revision, String>)t_property; + List<Revision> revs = new ArrayList<>(); + for (Map.Entry<Revision, String> en : pMap.entrySet()) { + Revision r = en.getKey(); + String bcv = _bc.get(r); + if ("true".equals(bcv)) { + revs.add(r); + } + } + // sort by age + Collections.sort(revs, new Comparator<Revision>() { + @Override + public int compare(Revision r1, Revision r2) { + if (r1.getClusterId() != r2.getClusterId()) { + return r1.getClusterId() - r2.getClusterId(); + } else if (r1.getTimestamp() != r2.getTimestamp()) { + return r1.getTimestamp() > r2.getTimestamp() ? 1 : -1; + } else { + return r1.getCounter() - r2.getCounter(); + } + }}); + + UpdateOp clean = new UpdateOp(doc.getId(), false); + Revision last = null; + for (Revision r : revs) { + if (last != null) { + if (last.getClusterId() == r.getClusterId()) { + clean.setMapEntry(propertyName, last, replacementValue); + } + } + last = r; + } + return clean.hasChanges() ? clean : null; + } else { + return null; + } + } + /** * List of property names that are system-defined by JCR and thus do not * need to be redacted (to be expanded later)
