Author: adulceanu
Date: Tue Mar 5 11:55:11 2019
New Revision: 1854844
URL: http://svn.apache.org/viewvc?rev=1854844&view=rev
Log:
OAK-8006 - SegmentBlob#readLongBlobId might cause SegmentNotFoundException on
standby
Merge revision 1852612 from trunk
Added:
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java
(with props)
Modified:
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java
Modified:
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java?rev=1854844&r1=1854843&r2=1854844&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
Tue Mar 5 11:55:11 2019
@@ -470,6 +470,7 @@ public class SegmentNodeStoreService {
if (configuration.isStandbyInstance()) {
builder.withSnfeListener(IGNORE_SNFE);
+ builder.withEagerSegmentCaching(true);
}
final FileStore store;
@@ -810,7 +811,7 @@ class Configuration {
/**
* Chooses repository home directory name based on <b>repository.home</b>
* property, defaulting to <b>repository</b> if property is not set.
- *
+ *
* @return repository home directory name.
*/
String getRepositoryHome() {
@@ -824,7 +825,7 @@ class Configuration {
/**
* Creates a new sub-directory relative to {@link #getRepositoryHome()} for
* storing segments.
- *
+ *
* @return directory for storing segments.
*/
File getSegmentDirectory() {
@@ -832,9 +833,9 @@ class Configuration {
}
/**
- * Creates a new sub-directory relative to {@link #getRepositoryHome()}
for
+ * Creates a new sub-directory relative to {@link #getRepositoryHome()} for
* storing repository backups.
- *
+ *
* @return directory for storing repository backups.
*/
File getBackupDirectory() {
Modified:
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1854844&r1=1854843&r2=1854844&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
Tue Mar 5 11:55:11 2019
@@ -161,6 +161,8 @@ public class FileStore extends AbstractF
@NotNull
private final SegmentNotFoundExceptionListener snfeListener;
+ private final boolean eagerSegmentCaching;
+
FileStore(final FileStoreBuilder builder) throws
InvalidFileStoreVersionException, IOException {
super(builder);
@@ -200,6 +202,7 @@ public class FileStore extends AbstractF
this.stats.init(size);
this.snfeListener = builder.getSnfeListener();
+ this.eagerSegmentCaching = builder.getEagerSegmentCaching();
TimerStats flushTimer =
builder.getStatsProvider().getTimer("oak.segment.flush", METRICS_ONLY);
fileStoreScheduler.scheduleAtFixedRate(format("TarMK flush [%s]",
directory), 5, SECONDS, () -> {
@@ -515,6 +518,11 @@ public class FileStore extends AbstractF
}
segment = new Segment(tracker, segmentReader, id, data);
+
+ if (eagerSegmentCaching) {
+ segmentCache.putSegment(segment);
+ }
+
generation = segment.getGcGeneration();
references = readReferences(segment);
binaryReferences = readBinaryReferences(segment);
@@ -531,7 +539,7 @@ public class FileStore extends AbstractF
);
// Keep this data segment in memory as it's likely to be accessed
soon.
- if (segment != null) {
+ if (!eagerSegmentCaching && segment != null) {
segmentCache.putSegment(segment);
}
}
Modified:
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java?rev=1854844&r1=1854843&r2=1854844&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
Tue Mar 5 11:55:11 2019
@@ -122,7 +122,9 @@ public class FileStoreBuilder {
private IOMonitor ioMonitor = new IOMonitorAdapter();
private boolean strictVersionCheck;
-
+
+ private boolean eagerSegmentCaching;
+
private boolean built;
/**
@@ -312,7 +314,20 @@ public class FileStoreBuilder {
this.strictVersionCheck = strictVersionCheck;
return this;
}
-
+
+ /**
+ * Enable eager segment caching. This proves useful when segments need to
+ * be cached as soon as they are created, right before persisting them to
disk.
+ * One such scenario is the cold standby, see OAK-8006.
+ *
+ * @param eagerSegmentCaching enables eager segment caching iff {@code
true}.
+ * @return this instance
+ */
+ public FileStoreBuilder withEagerSegmentCaching(boolean
eagerSegmentCaching) {
+ this.eagerSegmentCaching = eagerSegmentCaching;
+ return this;
+ }
+
/**
* Create a new {@link FileStore} instance with the settings specified in
this
* builder. If none of the {@code with} methods have been called before
calling
@@ -436,7 +451,7 @@ public class FileStoreBuilder {
SegmentGCOptions getGcOptions() {
return gcOptions;
}
-
+
@NotNull
SegmentNotFoundExceptionListener getSnfeListener() {
return snfeListener;
@@ -467,6 +482,10 @@ public class FileStoreBuilder {
return strictVersionCheck;
}
+ boolean getEagerSegmentCaching() {
+ return eagerSegmentCaching;
+ }
+
@Override
public String toString() {
return "FileStoreBuilder{" +
Added:
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java?rev=1854844&view=auto
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java
(added)
+++
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java
Tue Mar 5 11:55:11 2019
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.standby;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.commons.io.input.NullInputStream;
+import org.apache.jackrabbit.core.data.FileDataStore;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.junit.TemporaryPort;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.SegmentTestConstants;
+import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync;
+import org.apache.jackrabbit.oak.segment.standby.server.StandbyServerSync;
+import org.apache.jackrabbit.oak.segment.test.TemporaryBlobStore;
+import org.apache.jackrabbit.oak.segment.test.TemporaryFileStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+
+public class StandbySegmentBlobTestIT extends TestBase {
+
+ // `BLOB_SIZE` has to be chosen in such a way that is both:
+ //
+ // - Less than or equal to `minRecordLength` in the Blob Store. This is
+ // necessary in order to create a sufficiently long blob ID.
+ // - Greater than or equal to `Segment.BLOB_ID_SMALL_LIMIT` in order to
save
+ // the blob ID as a large, external blob ID.
+ // - Greater than or equal to `Segment.MEDIUM_LIMIT`, otherwise the content
+ // of the binary will be written as a mere value record instead of a
+ // binary ID referencing an external binary.
+ //
+ // Since `Segment.MEDIUM_LIMIT` is the largest of the constants above, it
is
+ // sufficient to set `BLOB_SIZE` to `Segment.MEDIUM_LIMIT`.
+
+ private static final int BLOB_SIZE = 2 * SegmentTestConstants.MEDIUM_LIMIT;
+
+ private TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ private TemporaryBlobStore blobStore = new TemporaryBlobStore(folder) {
+
+ @Override
+ protected void configureDataStore(FileDataStore dataStore) {
+ dataStore.setMinRecordLength(BLOB_SIZE);
+ }
+
+ };
+
+ private TemporaryFileStore serverFileStore = new
TemporaryFileStore(folder, blobStore, false);
+
+ private TemporaryFileStore clientFileStore = new
TemporaryFileStore(folder, blobStore, true);
+
+ @Rule
+ public TemporaryPort serverPort = new TemporaryPort();
+
+ @Rule
+ public RuleChain chain = RuleChain.outerRule(folder)
+ .around(blobStore)
+ .around(serverFileStore)
+ .around(clientFileStore);
+
+ @Test
+ public void testSyncWithLongBlobId() throws Exception {
+ FileStore primary = serverFileStore.fileStore();
+ FileStore secondary = clientFileStore.fileStore();
+
+ NodeStore store = SegmentNodeStoreBuilders.builder(primary).build();
+ try (
+ StandbyServerSync serverSync = new
StandbyServerSync(serverPort.getPort(), primary, MB);
+ StandbyClientSync clientSync = new
StandbyClientSync(getServerHost(), serverPort.getPort(), secondary, false,
getClientTimeout(), false, folder.newFolder())
+ ) {
+ serverSync.start();
+ addTestContent(store, "server");
+ primary.flush();
+
+ clientSync.run();
+
+ assertEquals(primary.getHead(), secondary.getHead());
+
+ PropertyState ps = secondary.getHead().getChildNode("root")
+ .getChildNode("server").getProperty("testBlob");
+ assertNotNull(ps);
+ assertEquals(Type.BINARY.tag(), ps.getType().tag());
+ Blob b = ps.getValue(Type.BINARY);
+ assertEquals(BLOB_SIZE, b.length());
+ }
+ }
+
+ private static void addTestContent(NodeStore store, String child) throws
Exception {
+ NodeBuilder builder = store.getRoot().builder();
+ NodeBuilder content = builder.child(child);
+ content.setProperty("ts", System.currentTimeMillis());
+
+ Blob blob = store.createBlob(new NullInputStream(BLOB_SIZE));
+ content.setProperty("testBlob", blob);
+
+ store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ }
+
+}
Propchange:
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/StandbySegmentBlobTestIT.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified:
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java?rev=1854844&r1=1854843&r2=1854844&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java
(original)
+++
jackrabbit/oak/branches/1.8/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java
Tue Mar 5 11:55:11 2019
@@ -36,7 +36,7 @@ public class TemporaryFileStore extends
private final TemporaryFolder folder;
private final TemporaryBlobStore blobStore;
-
+
private final boolean standby;
private ScheduledExecutorService executor;
@@ -62,7 +62,7 @@ public class TemporaryFileStore extends
.withMaxFileSize(1)
.withMemoryMapping(false)
.withNodeDeduplicationCacheSize(1)
- .withSegmentCacheSize(0)
+ .withSegmentCacheSize(256)
.withStringCacheSize(0)
.withTemplateCacheSize(0)
.withStatisticsProvider(new
DefaultStatisticsProvider(executor));
@@ -71,7 +71,8 @@ public class TemporaryFileStore extends
.setRetainedGenerations(1);
builder
.withGCOptions(gcOptions)
-
.withSnfeListener(SegmentNotFoundExceptionListener.IGNORE_SNFE);
+ .withSnfeListener(SegmentNotFoundExceptionListener.IGNORE_SNFE)
+ .withEagerSegmentCaching(true);
}
if (blobStore != null) {
builder.withBlobStore(blobStore.blobStore());
