Author: adulceanu Date: Thu May 7 10:53:45 2020 New Revision: 1877469 URL: http://svn.apache.org/viewvc?rev=1877469&view=rev Log: OAK-8832 - Offline Compaction fails while erroneously accessing external blob ((merged r1877063 into 1.10)
Added: jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorExternalBlobTest.java jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorTestUtils.java Modified: jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorTest.java Modified: jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java?rev=1877469&r1=1877468&r2=1877469&view=diff ============================================================================== --- jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java (original) +++ jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java Thu May 7 10:53:45 2020 @@ -176,6 +176,16 @@ public class SegmentBlob extends Record if (object instanceof SegmentBlob) { SegmentBlob that = (SegmentBlob) object; + if (blobStore == null) { + if (this.getContentIdentity() != null && that.getContentIdentity() != null) { + return this.getContentIdentity().equals(that.getContentIdentity()); + } + + if (this.isExternal() && !that.isExternal() || !this.isExternal() && that.isExternal()) { + return false; + } + } + if (this.length() != that.length()) { return false; } Added: jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorExternalBlobTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorExternalBlobTest.java?rev=1877469&view=auto ============================================================================== --- jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorExternalBlobTest.java (added) +++ jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorExternalBlobTest.java Thu May 7 10:53:45 2020 @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.jackrabbit.oak.segment; + +import static java.util.concurrent.TimeUnit.DAYS; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.addTestContent; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.assertSameRecord; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.assertSameStableId; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.checkGeneration; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.createBlob; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.createCompactor; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.getCheckpoint; +import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; +import static org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; +import org.apache.jackrabbit.oak.segment.file.cancel.Canceller; +import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration; +import org.apache.jackrabbit.oak.segment.test.TemporaryBlobStore; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.RuleChain; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; + +public class CheckpointCompactorExternalBlobTest { + + private TemporaryFolder folder = new TemporaryFolder(new File("target")); + + private TemporaryBlobStore tempoararyBlobStore = new TemporaryBlobStore(folder); + + private FileStore fileStore; + + private SegmentNodeStore nodeStore; + + private CheckpointCompactor compactor; + + private GCGeneration compactedGeneration; + + @Rule + public RuleChain rules = RuleChain.outerRule(folder) + .around(tempoararyBlobStore); + + public void setup(boolean withBlobStore) throws IOException, InvalidFileStoreVersionException { + BlobStore blobStore = tempoararyBlobStore.blobStore(); + FileStoreBuilder fileStoreBuilder = fileStoreBuilder(folder.getRoot()); + + if (withBlobStore) { + fileStoreBuilder = fileStoreBuilder.withBlobStore(blobStore); + } + + fileStore = fileStoreBuilder.build(); + nodeStore = SegmentNodeStoreBuilders.builder(fileStore).build(); + compactedGeneration = newGCGeneration(1,1, true); + compactor = createCompactor(fileStore, compactedGeneration); + } + + @After + public void tearDown() { + fileStore.close(); + } + + @Test + public void testCompact() throws Exception { + setup(true); + + // add two blobs which will be persisted in the blob store + addTestContent("cp1", nodeStore, SegmentTestConstants.MEDIUM_LIMIT); + String cp1 = nodeStore.checkpoint(DAYS.toMillis(1)); + addTestContent("cp2", nodeStore, SegmentTestConstants.MEDIUM_LIMIT); + String cp2 = nodeStore.checkpoint(DAYS.toMillis(1)); + + // update the two blobs from the blob store + updateTestContent("cp1", nodeStore); + String cp3 = nodeStore.checkpoint(DAYS.toMillis(1)); + updateTestContent("cp2", nodeStore); + String cp4 = nodeStore.checkpoint(DAYS.toMillis(1)); + fileStore.close(); + + // no blob store configured + setup(false); + + // this time the updated blob will be stored in the file store + updateTestContent("cp2", nodeStore); + String cp5 = nodeStore.checkpoint(DAYS.toMillis(1)); + + SegmentNodeState uncompacted1 = fileStore.getHead(); + SegmentNodeState compacted1 = compactor.compact(EMPTY_NODE, uncompacted1, EMPTY_NODE, Canceller.newCanceller()); + + assertNotNull(compacted1); + assertFalse(uncompacted1 == compacted1); + checkGeneration(compacted1, compactedGeneration); + + assertSameStableId(uncompacted1, compacted1); + assertSameStableId(getCheckpoint(uncompacted1, cp1), getCheckpoint(compacted1, cp1)); + assertSameStableId(getCheckpoint(uncompacted1, cp2), getCheckpoint(compacted1, cp2)); + assertSameStableId(getCheckpoint(uncompacted1, cp3), getCheckpoint(compacted1, cp3)); + assertSameStableId(getCheckpoint(uncompacted1, cp4), getCheckpoint(compacted1, cp4)); + assertSameStableId(getCheckpoint(uncompacted1, cp5), getCheckpoint(compacted1, cp5)); + assertSameRecord(getCheckpoint(compacted1, cp5), compacted1.getChildNode("root")); + } + + private static void updateTestContent(@NotNull String parent, @NotNull NodeStore nodeStore) + throws CommitFailedException, IOException { + NodeBuilder rootBuilder = nodeStore.getRoot().builder(); + NodeBuilder parentBuilder = rootBuilder.child(parent); + parentBuilder.child("b").setProperty("bin", createBlob(nodeStore, SegmentTestConstants.MEDIUM_LIMIT)); + nodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + } + +} \ No newline at end of file Modified: jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorTest.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorTest.java?rev=1877469&r1=1877468&r2=1877469&view=diff ============================================================================== --- jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorTest.java (original) +++ jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorTest.java Thu May 7 10:53:45 2020 @@ -18,11 +18,14 @@ package org.apache.jackrabbit.oak.segment; -import static com.google.common.collect.Lists.newArrayList; import static java.util.concurrent.TimeUnit.DAYS; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; -import static org.apache.jackrabbit.oak.plugins.memory.MultiBinaryPropertyState.binaryPropertyFromBlob; -import static org.apache.jackrabbit.oak.segment.DefaultSegmentWriterBuilder.defaultSegmentWriterBuilder; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.addTestContent; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.assertSameRecord; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.assertSameStableId; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.checkGeneration; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.createCompactor; +import static org.apache.jackrabbit.oak.segment.CheckpointCompactorTestUtils.getCheckpoint; import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; import static org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration; import static org.junit.Assert.assertEquals; @@ -30,28 +33,13 @@ import static org.junit.Assert.assertFal import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; -import java.util.List; -import java.util.Random; -import com.google.common.base.Suppliers; -import org.apache.jackrabbit.oak.api.Blob; -import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.segment.file.FileStore; -import org.apache.jackrabbit.oak.segment.file.GCNodeWriteMonitor; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; import org.apache.jackrabbit.oak.segment.file.cancel.Canceller; import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration; -import org.apache.jackrabbit.oak.spi.commit.CommitInfo; -import org.apache.jackrabbit.oak.spi.commit.EmptyHook; -import org.apache.jackrabbit.oak.spi.gc.GCMonitor; -import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; -import org.apache.jackrabbit.oak.spi.state.NodeBuilder; -import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.jackrabbit.oak.spi.state.NodeStore; -import org.jetbrains.annotations.NotNull; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -85,9 +73,9 @@ public class CheckpointCompactorTest { @Test public void testCompact() throws Exception { - addTestContent("cp1", nodeStore); + addTestContent("cp1", nodeStore, 42); String cp1 = nodeStore.checkpoint(DAYS.toMillis(1)); - addTestContent("cp2", nodeStore); + addTestContent("cp2", nodeStore, 42); String cp2 = nodeStore.checkpoint(DAYS.toMillis(1)); SegmentNodeState uncompacted1 = fileStore.getHead(); @@ -102,9 +90,9 @@ public class CheckpointCompactorTest { assertSameRecord(getCheckpoint(compacted1, cp2), compacted1.getChildNode("root")); // Simulate a 2nd compaction cycle - addTestContent("cp3", nodeStore); + addTestContent("cp3", nodeStore, 42); String cp3 = nodeStore.checkpoint(DAYS.toMillis(1)); - addTestContent("cp4", nodeStore); + addTestContent("cp4", nodeStore, 42); String cp4 = nodeStore.checkpoint(DAYS.toMillis(1)); SegmentNodeState uncompacted2 = fileStore.getHead(); @@ -125,80 +113,4 @@ public class CheckpointCompactorTest { assertSameRecord(getCheckpoint(compacted1, cp2), getCheckpoint(compacted2, cp2)); assertSameRecord(getCheckpoint(compacted2, cp4), compacted2.getChildNode("root")); } - - private static void checkGeneration(NodeState node, GCGeneration gcGeneration) { - assertTrue(node instanceof SegmentNodeState); - assertEquals(gcGeneration, ((SegmentNodeState) node).getRecordId().getSegmentId().getGcGeneration()); - - for (ChildNodeEntry cne : node.getChildNodeEntries()) { - checkGeneration(cne.getNodeState(), gcGeneration); - } - } - - private static NodeState getCheckpoint(NodeState superRoot, String name) { - NodeState checkpoint = superRoot - .getChildNode("checkpoints") - .getChildNode(name) - .getChildNode("root"); - assertTrue(checkpoint.exists()); - return checkpoint; - } - - private static void assertSameStableId(NodeState node1, NodeState node2) { - assertTrue(node1 instanceof SegmentNodeState); - assertTrue(node2 instanceof SegmentNodeState); - - assertEquals("Nodes should have the same stable ids", - ((SegmentNodeState) node1).getStableId(), - ((SegmentNodeState) node2).getStableId()); - } - - private static void assertSameRecord(NodeState node1, NodeState node2) { - assertTrue(node1 instanceof SegmentNodeState); - assertTrue(node2 instanceof SegmentNodeState); - - assertEquals("Nodes should have been deduplicated", - ((SegmentNodeState) node1).getRecordId(), - ((SegmentNodeState) node2).getRecordId()); - } - - @NotNull - private static CheckpointCompactor createCompactor(@NotNull FileStore fileStore, @NotNull GCGeneration generation) { - SegmentWriter writer = defaultSegmentWriterBuilder("c") - .withGeneration(generation) - .build(fileStore); - - return new CheckpointCompactor( - GCMonitor.EMPTY, - fileStore.getReader(), - writer, - fileStore.getBlobStore(), - GCNodeWriteMonitor.EMPTY); - } - - private static void addTestContent(@NotNull String parent, @NotNull NodeStore nodeStore) - throws CommitFailedException, IOException { - NodeBuilder rootBuilder = nodeStore.getRoot().builder(); - NodeBuilder parentBuilder = rootBuilder.child(parent); - parentBuilder.setChildNode("a").setChildNode("aa").setProperty("p", 42); - parentBuilder.getChildNode("a").setChildNode("bb").setChildNode("bbb"); - parentBuilder.setChildNode("b").setProperty("bin", createBlob(nodeStore, 42)); - parentBuilder.setChildNode("c").setProperty(binaryPropertyFromBlob("bins", createBlobs(nodeStore, 42, 43, 44))); - nodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); - } - - private static Blob createBlob(NodeStore nodeStore, int size) throws IOException { - byte[] data = new byte[size]; - new Random().nextBytes(data); - return nodeStore.createBlob(new ByteArrayInputStream(data)); - } - - private static List<Blob> createBlobs(NodeStore nodeStore, int... sizes) throws IOException { - List<Blob> blobs = newArrayList(); - for (int size : sizes) { - blobs.add(createBlob(nodeStore, size)); - } - return blobs; - } - } Added: jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorTestUtils.java URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorTestUtils.java?rev=1877469&view=auto ============================================================================== --- jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorTestUtils.java (added) +++ jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CheckpointCompactorTestUtils.java Thu May 7 10:53:45 2020 @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.jackrabbit.oak.segment; + +import static com.google.common.collect.Lists.newArrayList; +import static org.apache.jackrabbit.oak.plugins.memory.MultiBinaryPropertyState.binaryPropertyFromBlob; +import static org.apache.jackrabbit.oak.segment.DefaultSegmentWriterBuilder.defaultSegmentWriterBuilder; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.file.GCNodeWriteMonitor; +import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.gc.GCMonitor; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.jetbrains.annotations.NotNull; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.util.List; +import java.util.Random; + +public class CheckpointCompactorTestUtils { + + private CheckpointCompactorTestUtils() { + + } + + public static void checkGeneration(NodeState node, GCGeneration gcGeneration) { + assertTrue(node instanceof SegmentNodeState); + assertEquals(gcGeneration, ((SegmentNodeState) node).getRecordId().getSegmentId().getGcGeneration()); + + for (ChildNodeEntry cne : node.getChildNodeEntries()) { + checkGeneration(cne.getNodeState(), gcGeneration); + } + } + + public static NodeState getCheckpoint(NodeState superRoot, String name) { + NodeState checkpoint = superRoot + .getChildNode("checkpoints") + .getChildNode(name) + .getChildNode("root"); + assertTrue(checkpoint.exists()); + return checkpoint; + } + + public static void assertSameStableId(NodeState node1, NodeState node2) { + assertTrue(node1 instanceof SegmentNodeState); + assertTrue(node2 instanceof SegmentNodeState); + + assertEquals("Nodes should have the same stable ids", + ((SegmentNodeState) node1).getStableId(), + ((SegmentNodeState) node2).getStableId()); + } + + public static void assertSameRecord(NodeState node1, NodeState node2) { + assertTrue(node1 instanceof SegmentNodeState); + assertTrue(node2 instanceof SegmentNodeState); + + assertEquals("Nodes should have been deduplicated", + ((SegmentNodeState) node1).getRecordId(), + ((SegmentNodeState) node2).getRecordId()); + } + + @NotNull + public static CheckpointCompactor createCompactor(@NotNull FileStore fileStore, @NotNull GCGeneration generation) { + SegmentWriter writer = defaultSegmentWriterBuilder("c") + .withGeneration(generation) + .build(fileStore); + + return new CheckpointCompactor( + GCMonitor.EMPTY, + fileStore.getReader(), + writer, + fileStore.getBlobStore(), + GCNodeWriteMonitor.EMPTY); + } + + public static void addTestContent(@NotNull String parent, @NotNull NodeStore nodeStore, int binPropertySize) + throws CommitFailedException, IOException { + NodeBuilder rootBuilder = nodeStore.getRoot().builder(); + NodeBuilder parentBuilder = rootBuilder.child(parent); + parentBuilder.setChildNode("a").setChildNode("aa").setProperty("p", 42); + parentBuilder.getChildNode("a").setChildNode("bb").setChildNode("bbb"); + parentBuilder.setChildNode("b").setProperty("bin", createBlob(nodeStore, binPropertySize)); + parentBuilder.setChildNode("c").setProperty(binaryPropertyFromBlob("bins", createBlobs(nodeStore, 42, 43, 44))); + nodeStore.merge(rootBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + } + + public static Blob createBlob(NodeStore nodeStore, int size) throws IOException { + byte[] data = new byte[size]; + new Random().nextBytes(data); + return nodeStore.createBlob(new ByteArrayInputStream(data)); + } + + public static List<Blob> createBlobs(NodeStore nodeStore, int... sizes) throws IOException { + List<Blob> blobs = newArrayList(); + for (int size : sizes) { + blobs.add(createBlob(nodeStore, size)); + } + return blobs; + } +}