Author: frm
Date: Tue Mar 5 14:30:14 2019
New Revision: 1854861
URL: http://svn.apache.org/viewvc?rev=1854861&view=rev
Log:
OAK-6749 - Don't trigger a binary download request for in-memory Blob IDs
Added:
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobFetchTimeoutException.java
- copied unchanged from r1851533,
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobFetchTimeoutException.java
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobProcessor.java
- copied unchanged from r1851533,
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobProcessor.java
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobTypeUnknownException.java
- copied unchanged from r1851533,
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobTypeUnknownException.java
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobWriteException.java
- copied unchanged from r1851533,
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/BlobWriteException.java
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
- copied, changed from r1851535,
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/
- copied from r1851534,
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/
Modified:
jackrabbit/oak/branches/1.10/ (props changed)
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java
jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
Propchange: jackrabbit/oak/branches/1.10/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 5 14:30:14 2019
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1850874,1850882,1851236,1851253,1851451,1852052,1852084,1852120,1852451,1852492-1852493,1852528,1852582,1852584,1852601,1852920,1853141,1853229,1853393,1853429,1853433,1853441,1853866,1853868,1853870,1853893,1853969,1853997,1854034,1854044,1854058,1854113,1854373,1854377,1854380,1854385,1854401,1854403,1854461-1854462,1854466,1854468,1854515,1854533
+/jackrabbit/oak/trunk:1850874,1850882,1851236,1851253,1851451,1851533-1851535,1851619,1852052,1852084,1852120,1852451,1852492-1852493,1852528,1852582,1852584,1852601,1852920,1853141,1853229,1853393,1853429,1853433,1853441,1853866,1853868,1853870,1853893,1853969,1853997,1854034,1854044,1854058,1854113,1854373,1854377,1854380,1854385,1854401,1854403,1854461-1854462,1854466,1854468,1854515,1854533
/jackrabbit/trunk:1345480
Copied:
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
(from r1851535,
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java)
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java?p2=jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java&p1=jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java&r1=1851535&r2=1854861&rev=1854861&view=diff
==============================================================================
---
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
(original)
+++
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessor.java
Tue Mar 5 14:30:14 2019
@@ -19,6 +19,8 @@
package org.apache.jackrabbit.oak.segment.standby.client;
+import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
+
import java.io.IOException;
import java.io.InputStream;
@@ -59,7 +61,59 @@ class RemoteBlobProcessor implements Blo
}
private boolean shouldFetchBinary(SegmentBlob blob) {
- return blob.isExternal() && blob.getReference() == null &&
blob.getBlobId() != null;
+
+ // Shortcut: If the Blob ID is null, this is an inline binary and we
+ // don't have to fetch it.
+
+ String blobId = blob.getBlobId();
+
+ if (blobId == null) {
+ return false;
+ }
+
+ // Shortcut: If the Blob Store is able to retrieve a non-null reference
+ // to the Blob, we can be sure that the Blob is already stored locally.
+ // We don't have to download it.
+
+ String reference;
+
+ try {
+ reference = blob.getReference();
+ } catch (Exception e) {
+ reference = null;
+ }
+
+ if (reference != null) {
+ return false;
+ }
+
+ // Worst case: A null reference to the Blob might just mean that the
+ // Blob Store doesn't support references. The Blob might still be
stored
+ // locally. We have to retrieve an InputStream for the Blob, and
+ // perform a tentative read in order to overcome a possible lazy
+ // implementation of the returned InputStream.
+
+ InputStream data;
+
+ try {
+ data = blobStore.getInputStream(blobId);
+ } catch (Exception e) {
+ return true;
+ }
+
+ if (data == null) {
+ return true;
+ }
+
+ try {
+ data.read();
+ } catch (Exception e) {
+ return true;
+ } finally {
+ closeQuietly(data);
+ }
+
+ return false;
}
private void fetchAndStoreBlob(String blobId) throws InterruptedException {
Modified:
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java?rev=1854861&r1=1854860&r2=1854861&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java
(original)
+++
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyDiff.java
Tue Mar 5 14:30:14 2019
@@ -31,7 +31,6 @@ import org.apache.jackrabbit.oak.api.Blo
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.segment.CancelableDiff;
-import org.apache.jackrabbit.oak.segment.SegmentBlob;
import org.apache.jackrabbit.oak.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
@@ -51,24 +50,30 @@ class StandbyDiff implements NodeStateDi
private final StandbyClient client;
- private final boolean hasDataStore;
-
private final String path;
private final Supplier<Boolean> running;
+ private final BlobProcessor blobProcessor;
+
StandbyDiff(NodeBuilder builder, FileStore store, StandbyClient client,
Supplier<Boolean> running) {
this(builder, store, client, "/", running);
}
- private StandbyDiff(NodeBuilder builder, FileStore store, StandbyClient
client, String path,
- Supplier<Boolean> running) {
+ private static BlobProcessor newBinaryFetcher(BlobStore blobStore,
StandbyClient client) {
+ if (blobStore == null) {
+ return (blob) -> {};
+ }
+ return new RemoteBlobProcessor(blobStore, client::getBlob);
+ }
+
+ private StandbyDiff(NodeBuilder builder, FileStore store, StandbyClient
client, String path, Supplier<Boolean> running) {
this.builder = builder;
this.store = store;
- this.hasDataStore = store.getBlobStore() != null;
this.client = client;
this.path = path;
this.running = running;
+ this.blobProcessor = newBinaryFetcher(store.getBlobStore(), client);
}
@Override
@@ -129,7 +134,7 @@ class StandbyDiff implements NodeStateDi
return (SegmentNodeState) after;
}
- if (!hasDataStore) {
+ if (store.getBlobStore() == null) {
return (SegmentNodeState) after;
}
@@ -137,7 +142,7 @@ class StandbyDiff implements NodeStateDi
// traversal to verify binaries
for (PropertyState propertyState : after.getProperties()) {
- fetchBinary(propertyState);
+ processBinary(propertyState);
}
boolean success = after.compareAgainstBaseState(before, new
CancelableDiff(this, newCanceledSupplier()));
@@ -162,59 +167,44 @@ class StandbyDiff implements NodeStateDi
};
}
- private PropertyState fetchBinary(PropertyState property) {
+ private PropertyState processBinary(PropertyState property) {
Type<?> type = property.getType();
if (type == BINARY) {
- fetchBinary(property.getValue(Type.BINARY), property.getName());
+ processBinary(property.getValue(Type.BINARY), property.getName());
} else if (type == BINARIES) {
for (Blob blob : property.getValue(BINARIES)) {
- fetchBinary(blob, property.getName());
+ processBinary(blob, property.getName());
}
}
return property;
}
- private void fetchBinary(Blob b, String pName) {
- if (b instanceof SegmentBlob) {
- fetchBinary((SegmentBlob) b, pName);
- } else {
- log.warn("Unknown Blob {} at {}, ignoring",
b.getClass().getName(), path + "#" + pName);
- }
- }
-
- private void fetchBinary(SegmentBlob sb, String pName) {
- if (sb.isExternal() && hasDataStore && sb.getReference() == null) {
- String blobId = sb.getBlobId();
-
- if (blobId == null) {
- return;
- }
-
- try {
- fetchAndStoreBlob(blobId, pName);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- private void fetchAndStoreBlob(String blobId, String pName) throws
InterruptedException {
- InputStream in = client.getBlob(blobId);
-
- if (in == null) {
- throw new IllegalStateException("Unable to load remote blob " +
blobId + " at " + path + "#" + pName
- + " in " + client.getReadTimeoutMs() + "ms. Please
increase the timeout and try again.");
- }
-
+ private void processBinary(Blob b, String propertyName) {
try {
- BlobStore blobStore = store.getBlobStore();
- assert blobStore != null : "Blob store must not be null";
- blobStore.writeBlob(in);
- in.close();
- } catch (IOException f) {
- throw new IllegalStateException("Unable to persist blob " + blobId
+ " at " + path + "#" + pName, f);
+ blobProcessor.processBinary(b);
+ } catch (BlobFetchTimeoutException e) {
+ String message = String.format(
+ "Unable to load remote blob %s at %s#%s in %dms. Please
increase the timeout and try again.",
+ e.getBlobId(),
+ path,
+ propertyName,
+ client.getReadTimeoutMs()
+ );
+ throw new IllegalStateException(message, e);
+ } catch (BlobWriteException e) {
+ String message = String.format(
+ "Unable to persist blob %s at %s#%s",
+ e.getBlobId(),
+ path,
+ propertyName
+ );
+ throw new IllegalStateException(message, e);
+ } catch (BlobTypeUnknownException e) {
+ log.warn("Unknown Blob {} at {}, ignoring",
b.getClass().getName(), path + "#" + propertyName);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
}
}
Modified:
jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java?rev=1854861&r1=1851534&r2=1854861&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
(original)
+++
jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java
Tue Mar 5 14:30:14 2019
@@ -33,7 +33,6 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.RuleChain;
@@ -85,7 +84,6 @@ public class RemoteBlobProcessorTest {
* downloaded.
*/
@Test
- @Ignore("OAK-6749")
public void inMemoryBinaryShouldNotBeDownloaded() throws Exception {
SegmentNodeStore store =
SegmentNodeStoreBuilders.builder(fileStore.fileStore()).build();
Modified:
jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
URL:
http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java?rev=1854861&r1=1854860&r2=1854861&view=diff
==============================================================================
---
jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
(original)
+++
jackrabbit/oak/branches/1.10/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryBlobStore.java
Tue Mar 5 14:30:14 2019
@@ -37,7 +37,7 @@ public class TemporaryBlobStore extends
@Override
protected void before() throws Throwable {
FileDataStore fds = new FileDataStore();
- fds.setMinRecordLength(4092);
+ configureDataStore(fds);
fds.init(folder.newFolder().getAbsolutePath());
store = new DataStoreBlobStore(fds);
}
