[
https://issues.apache.org/jira/browse/OAK-4969?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Timothee Maret updated OAK-4969:
--------------------------------
Description:
1. In a setup composed of two instances (primary, standby) configured with a
custom blob store (File blob store).
2. On the primary instance, set/update a BINARY property of an existing
resource with > 2MB binary.
3. Observe that the standby instance does not fetch the binary and instead,
enters a loop detecting the missing binary upon comparing node states.
For example, the following stack trace would be printed every 5 seconds on the
standby (the polling time is 5sec).
{code}
19.10.2016 16:22:38.035 *DEBUG* [nioEventLoopGroup-1005-1]
org.apache.jackrabbit.oak.segment.standby.codec.ResponseDecoder Decoding 'get
head' response
19.10.2016 16:22:38.038 *DEBUG* [sling-default-81-Registered Service.607]
org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Channel closed
19.10.2016 16:22:40.241 *DEBUG* [sling-default-81-Registered Service.607]
org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Group shut down
19.10.2016 16:22:40.242 *ERROR* [sling-default-81-Registered Service.607]
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync Failed
synchronizing state.
java.lang.RuntimeException: Error occurred while obtaining InputStream for
blobId [4dfc748c91d518c9221031ec6115fd7ac04fe27b#100000]
at
org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:49)
at
org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:252)
at
org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:87)
at
org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:45)
at
org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:42)
at com.google.common.io.ByteStreams$3.openStream(ByteStreams.java:907)
at com.google.common.io.ByteSource.contentEquals(ByteSource.java:301)
at com.google.common.io.ByteStreams.equal(ByteStreams.java:661)
at
org.apache.jackrabbit.oak.plugins.memory.AbstractBlob.equal(AbstractBlob.java:68)
at
org.apache.jackrabbit.oak.segment.SegmentBlob.equals(SegmentBlob.java:193)
at com.google.common.base.Objects.equal(Objects.java:55)
at
org.apache.jackrabbit.oak.plugins.memory.AbstractPropertyState.equal(AbstractPropertyState.java:53)
at
org.apache.jackrabbit.oak.segment.SegmentPropertyState.equals(SegmentPropertyState.java:249)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareProperties(SegmentNodeState.java:622)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:516)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:415)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:457)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
at
org.apache.jackrabbit.oak.segment.MapRecord$2.childNodeChanged(MapRecord.java:401)
at
org.apache.jackrabbit.oak.segment.MapRecord$3.childNodeChanged(MapRecord.java:442)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:433)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:391)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.compareAgainstBaseState(StandbyClientSyncExecution.java:91)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.execute(StandbyClientSyncExecution.java:74)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync.run(StandbyClientSync.java:143)
at
org.apache.sling.commons.scheduler.impl.QuartzJobExecutor.execute(QuartzJobExecutor.java:118)
at org.quartz.core.JobRunShell.run(JobRunShell.java:202)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.io.IOException:
org.apache.jackrabbit.core.data.DataStoreException: Record
4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist
at
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:560)
at
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getInputStream(DataStoreBlobStore.java:330)
at
org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:47)
... 42 common frames omitted
Caused by: org.apache.jackrabbit.core.data.DataStoreException: Record
4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist
at
org.apache.jackrabbit.core.data.AbstractDataStore.getRecord(AbstractDataStore.java:53)
at
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getDataRecord(DataStoreBlobStore.java:569)
at
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:554)
... 44 common frames omitted
{code}
was:
1. In a setup composed of two instances (primary, standdy) configured with a
custom blob store (File blob store).
2. On the primary instance, set/update a BINARY property of an existing
resource with > 2MB binary.
3. Observe that the standby instance does not fetch the binary and instead,
enters a loop detecting the missing binary upon comparing node states.
For example, the following stack trace would be printed every 5 seconds on the
standby (the polling time is 5sec).
{code}
19.10.2016 16:22:38.035 *DEBUG* [nioEventLoopGroup-1005-1]
org.apache.jackrabbit.oak.segment.standby.codec.ResponseDecoder Decoding 'get
head' response
19.10.2016 16:22:38.038 *DEBUG* [sling-default-81-Registered Service.607]
org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Channel closed
19.10.2016 16:22:40.241 *DEBUG* [sling-default-81-Registered Service.607]
org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Group shut down
19.10.2016 16:22:40.242 *ERROR* [sling-default-81-Registered Service.607]
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync Failed
synchronizing state.
java.lang.RuntimeException: Error occurred while obtaining InputStream for
blobId [4dfc748c91d518c9221031ec6115fd7ac04fe27b#100000]
at
org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:49)
at
org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:252)
at
org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:87)
at
org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:45)
at
org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:42)
at com.google.common.io.ByteStreams$3.openStream(ByteStreams.java:907)
at com.google.common.io.ByteSource.contentEquals(ByteSource.java:301)
at com.google.common.io.ByteStreams.equal(ByteStreams.java:661)
at
org.apache.jackrabbit.oak.plugins.memory.AbstractBlob.equal(AbstractBlob.java:68)
at
org.apache.jackrabbit.oak.segment.SegmentBlob.equals(SegmentBlob.java:193)
at com.google.common.base.Objects.equal(Objects.java:55)
at
org.apache.jackrabbit.oak.plugins.memory.AbstractPropertyState.equal(AbstractPropertyState.java:53)
at
org.apache.jackrabbit.oak.segment.SegmentPropertyState.equals(SegmentPropertyState.java:249)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareProperties(SegmentNodeState.java:622)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:516)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:415)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:457)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
at
org.apache.jackrabbit.oak.segment.MapRecord$2.childNodeChanged(MapRecord.java:401)
at
org.apache.jackrabbit.oak.segment.MapRecord$3.childNodeChanged(MapRecord.java:442)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:433)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:391)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
at
org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485)
at
org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.compareAgainstBaseState(StandbyClientSyncExecution.java:91)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.execute(StandbyClientSyncExecution.java:74)
at
org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync.run(StandbyClientSync.java:143)
at
org.apache.sling.commons.scheduler.impl.QuartzJobExecutor.execute(QuartzJobExecutor.java:118)
at org.quartz.core.JobRunShell.run(JobRunShell.java:202)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.io.IOException:
org.apache.jackrabbit.core.data.DataStoreException: Record
4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist
at
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:560)
at
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getInputStream(DataStoreBlobStore.java:330)
at
org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:47)
... 42 common frames omitted
Caused by: org.apache.jackrabbit.core.data.DataStoreException: Record
4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist
at
org.apache.jackrabbit.core.data.AbstractDataStore.getRecord(AbstractDataStore.java:53)
at
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getDataRecord(DataStoreBlobStore.java:569)
at
org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:554)
... 44 common frames omitted
{code}
> ColdStandby does not fetch missing blobs
> ----------------------------------------
>
> Key: OAK-4969
> URL: https://issues.apache.org/jira/browse/OAK-4969
> Project: Jackrabbit Oak
> Issue Type: Bug
> Components: segment-tar
> Affects Versions: Segment Tar 0.0.10
> Reporter: Timothee Maret
> Fix For: Segment Tar 0.0.16
>
>
> 1. In a setup composed of two instances (primary, standby) configured with a
> custom blob store (File blob store).
> 2. On the primary instance, set/update a BINARY property of an existing
> resource with > 2MB binary.
> 3. Observe that the standby instance does not fetch the binary and instead,
> enters a loop detecting the missing binary upon comparing node states.
> For example, the following stack trace would be printed every 5 seconds on
> the standby (the polling time is 5sec).
> {code}
> 19.10.2016 16:22:38.035 *DEBUG* [nioEventLoopGroup-1005-1]
> org.apache.jackrabbit.oak.segment.standby.codec.ResponseDecoder Decoding 'get
> head' response
> 19.10.2016 16:22:38.038 *DEBUG* [sling-default-81-Registered Service.607]
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Channel closed
> 19.10.2016 16:22:40.241 *DEBUG* [sling-default-81-Registered Service.607]
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClient Group shut down
> 19.10.2016 16:22:40.242 *ERROR* [sling-default-81-Registered Service.607]
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync Failed
> synchronizing state.
> java.lang.RuntimeException: Error occurred while obtaining InputStream for
> blobId [4dfc748c91d518c9221031ec6115fd7ac04fe27b#100000]
> at
> org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:49)
> at
> org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:252)
> at
> org.apache.jackrabbit.oak.segment.SegmentBlob.getNewStream(SegmentBlob.java:87)
> at
> org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:45)
> at
> org.apache.jackrabbit.oak.plugins.memory.AbstractBlob$1.getInput(AbstractBlob.java:42)
> at com.google.common.io.ByteStreams$3.openStream(ByteStreams.java:907)
> at com.google.common.io.ByteSource.contentEquals(ByteSource.java:301)
> at com.google.common.io.ByteStreams.equal(ByteStreams.java:661)
> at
> org.apache.jackrabbit.oak.plugins.memory.AbstractBlob.equal(AbstractBlob.java:68)
> at
> org.apache.jackrabbit.oak.segment.SegmentBlob.equals(SegmentBlob.java:193)
> at com.google.common.base.Objects.equal(Objects.java:55)
> at
> org.apache.jackrabbit.oak.plugins.memory.AbstractPropertyState.equal(AbstractPropertyState.java:53)
> at
> org.apache.jackrabbit.oak.segment.SegmentPropertyState.equals(SegmentPropertyState.java:249)
> at
> org.apache.jackrabbit.oak.segment.SegmentNodeState.compareProperties(SegmentNodeState.java:622)
> at
> org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:516)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
> at
> org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:415)
> at
> org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
> at
> org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:457)
> at
> org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
> at
> org.apache.jackrabbit.oak.segment.MapRecord$2.childNodeChanged(MapRecord.java:401)
> at
> org.apache.jackrabbit.oak.segment.MapRecord$3.childNodeChanged(MapRecord.java:442)
> at
> org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485)
> at
> org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:433)
> at
> org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:391)
> at
> org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.process(StandbyDiff.java:216)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyDiff.childNodeChanged(StandbyDiff.java:186)
> at
> org.apache.jackrabbit.oak.segment.MapRecord.compare(MapRecord.java:485)
> at
> org.apache.jackrabbit.oak.segment.SegmentNodeState.compareAgainstBaseState(SegmentNodeState.java:609)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.compareAgainstBaseState(StandbyClientSyncExecution.java:91)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSyncExecution.execute(StandbyClientSyncExecution.java:74)
> at
> org.apache.jackrabbit.oak.segment.standby.client.StandbyClientSync.run(StandbyClientSync.java:143)
> at
> org.apache.sling.commons.scheduler.impl.QuartzJobExecutor.execute(QuartzJobExecutor.java:118)
> at org.quartz.core.JobRunShell.run(JobRunShell.java:202)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.io.IOException:
> org.apache.jackrabbit.core.data.DataStoreException: Record
> 4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist
> at
> org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:560)
> at
> org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getInputStream(DataStoreBlobStore.java:330)
> at
> org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob.getNewStream(BlobStoreBlob.java:47)
> ... 42 common frames omitted
> Caused by: org.apache.jackrabbit.core.data.DataStoreException: Record
> 4dfc748c91d518c9221031ec6115fd7ac04fe27b does not exist
> at
> org.apache.jackrabbit.core.data.AbstractDataStore.getRecord(AbstractDataStore.java:53)
> at
> org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getDataRecord(DataStoreBlobStore.java:569)
> at
> org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore.getStream(DataStoreBlobStore.java:554)
> ... 44 common frames omitted
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
