Author: mreutegg
Date: Thu Sep 29 20:14:56 2016
New Revision: 1762833
URL: http://svn.apache.org/viewvc?rev=1762833&view=rev
Log:
OAK-4867: Avoid queries for first level previous documents during GC
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
(with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java?rev=1762833&r1=1762832&r2=1762833&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
(original)
+++
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
Thu Sep 29 20:14:56 2016
@@ -29,7 +29,9 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.base.Stopwatch;
@@ -46,6 +48,7 @@ import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.StandardSystemProperty.LINE_SEPARATOR;
+import static com.google.common.collect.Iterables.all;
import static com.google.common.collect.Iterators.partition;
import static java.util.Collections.singletonMap;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
@@ -231,10 +234,7 @@ public class VersionGarbageCollector {
if (doc.getNodeAtRevision(nodeStore, headRevision, null) == null) {
addDocument(id);
// Collect id of all previous docs also
- Iterator<NodeDocument> it = doc.getAllPreviousDocs();
- while (it.hasNext()) {
- addPreviousDocument(it.next().getId());
- }
+ addPreviousDocuments(previousDocIdsFor(doc));
}
}
@@ -267,6 +267,35 @@ public class VersionGarbageCollector {
//------------------------------< internal
>----------------------------
+ private Iterator<String> previousDocIdsFor(NodeDocument doc) {
+ Map<Revision, Range> prevRanges = doc.getPreviousRanges(true);
+ if (prevRanges.isEmpty()) {
+ return Iterators.emptyIterator();
+ } else if (all(prevRanges.values(), FIRST_LEVEL)) {
+ // all previous document ids can be constructed from the
+ // previous ranges map. this works for first level previous
+ // documents only.
+ final String path = doc.getPath();
+ return Iterators.transform(prevRanges.entrySet().iterator(),
+ new Function<Map.Entry<Revision, Range>, String>() {
+ @Override
+ public String apply(Map.Entry<Revision, Range> input) {
+ int h = input.getValue().getHeight();
+ return Utils.getPreviousIdFor(path, input.getKey(), h);
+ }
+ });
+ } else {
+ // need to fetch the previous documents to get their ids
+ return Iterators.transform(doc.getAllPreviousDocs(),
+ new Function<NodeDocument, String>() {
+ @Override
+ public String apply(NodeDocument input) {
+ return input.getId();
+ }
+ });
+ }
+ }
+
private void addDocument(String id) throws IOException {
docIdsToDelete.add(id);
}
@@ -275,8 +304,10 @@ public class VersionGarbageCollector {
return prevDocIdsToDelete.getSize() - exclude.size();
}
- private void addPreviousDocument(String id) throws IOException {
- prevDocIdsToDelete.add(id);
+ private void addPreviousDocuments(Iterator<String> ids) throws
IOException {
+ while (ids.hasNext()) {
+ prevDocIdsToDelete.add(ids.next());
+ }
}
private Iterator<String> getDocIdsToDelete() throws IOException {
@@ -402,4 +433,11 @@ public class VersionGarbageCollector {
return new StringSort(overflowToDiskThreshold,
NodeDocumentIdComparator.INSTANCE);
}
+
+ private static final Predicate<Range> FIRST_LEVEL = new Predicate<Range>()
{
+ @Override
+ public boolean apply(@Nullable Range input) {
+ return input != null && input.height == 0;
+ }
+ };
}
Added:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java?rev=1762833&view=auto
==============================================================================
---
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
(added)
+++
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
Thu Sep 29 20:14:56 2016
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.io.InputStream;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
+import org.apache.jackrabbit.oak.plugins.document.memory.MemoryDocumentStore;
+import org.apache.jackrabbit.oak.plugins.document.util.Utils;
+import org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class VersionGCQueryTest {
+
+ @Rule
+ public final DocumentMKBuilderProvider provider = new
DocumentMKBuilderProvider();
+
+ private Clock clock;
+ private Set<String> prevDocIds = Sets.newHashSet();
+ private DocumentStore store;
+ private DocumentNodeStore ns;
+
+ @Before
+ public void before() throws Exception {
+ clock = new Clock.Virtual();
+ clock.waitUntil(System.currentTimeMillis());
+ Revision.setClock(clock);
+ store = new MemoryDocumentStore() {
+ @Override
+ public <T extends Document> T find(Collection<T> collection,
+ String key) {
+ if (collection == Collection.NODES &&
Utils.isPreviousDocId(key)) {
+ prevDocIds.add(key);
+ }
+ return super.find(collection, key);
+ }
+ };
+ ns = provider.newBuilder().setDocumentStore(store)
+ .setAsyncDelay(0).clock(clock).getNodeStore();
+ }
+
+ @After
+ public void after() {
+ Revision.resetClockToDefault();
+ }
+
+ @Test
+ public void noQueryForFirstLevelPrevDocs() throws Exception {
+ // create some garbage
+ NodeBuilder builder = ns.getRoot().builder();
+ for (int i = 0; i < 10; i++) {
+ InputStream s = new RandomStream(10 * 1024, 42);
+ PropertyState p = new BinaryPropertyState("p", ns.createBlob(s));
+ builder.child("test").child("node-" + i).setProperty(p);
+ }
+ merge(builder);
+ builder = ns.getRoot().builder();
+ builder.child("test").remove();
+ merge(builder);
+ ns.runBackgroundOperations();
+
+ clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(1));
+
+ VersionGarbageCollector gc = new VersionGarbageCollector(
+ ns, new VersionGCSupport(store));
+ prevDocIds.clear();
+ VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ assertEquals(11, stats.deletedDocGCCount);
+ assertEquals(10, stats.splitDocGCCount);
+ assertEquals(0, prevDocIds.size());
+ assertEquals(1, Iterables.size(Utils.getAllDocuments(store)));
+ }
+
+ @Test
+ public void queryDeepPreviousDocs() throws Exception {
+ // create garbage until we have intermediate previous docs
+ NodeBuilder builder = ns.getRoot().builder();
+ builder.child("test");
+ merge(builder);
+ String id = Utils.getIdFromPath("/test");
+ while (!Iterables.any(store.find(Collection.NODES,
id).getPreviousRanges().values(), INTERMEDIATE)) {
+ InputStream s = new RandomStream(10 * 1024, 42);
+ PropertyState p = new BinaryPropertyState("p", ns.createBlob(s));
+ builder = ns.getRoot().builder();
+ builder.child("test").setProperty(p);
+ merge(builder);
+ builder = ns.getRoot().builder();
+ builder.child("test").remove();
+ merge(builder);
+ ns.runBackgroundOperations();
+ }
+ int numPrevDocs = Iterators.size(store.find(Collection.NODES,
id).getAllPreviousDocs());
+
+ clock.waitUntil(clock.getTime() + TimeUnit.HOURS.toMillis(1));
+
+ VersionGarbageCollector gc = new VersionGarbageCollector(
+ ns, new VersionGCSupport(store));
+ prevDocIds.clear();
+ VersionGCStats stats = gc.gc(30, TimeUnit.MINUTES);
+ assertEquals(1, stats.deletedDocGCCount);
+ assertEquals(numPrevDocs, stats.splitDocGCCount);
+ assertEquals(numPrevDocs, prevDocIds.size());
+ assertEquals(1, Iterables.size(Utils.getAllDocuments(store)));
+ }
+
+ private NodeState merge(NodeBuilder builder) throws CommitFailedException {
+ return ns.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+
+ private static final Predicate<Range> INTERMEDIATE =
+ new Predicate<Range>() {
+ @Override
+ public boolean apply(Range input) {
+ return input.height > 0;
+ }
+ };
+}
Propchange:
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCQueryTest.java
------------------------------------------------------------------------------
svn:eol-style = native
