[
https://issues.apache.org/jira/browse/OAK-2961?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14583526#comment-14583526
]
Chetan Mehrotra commented on OAK-2961:
--------------------------------------
Looks like the issue is in AsyncIndexUpdate around what is considered as
concurrent update. The above test runs a 5 node cluster and creates and index
and triggers a reindex. It can happen and infact here does happen that on 2
cluster node AsyncIndex job is fired at same time and they both try to set the
lease which can result in conflict at DocumentStore layer (and hence not
detected by CommitHook). For e.g. in this test following logs is seen where it
can be seen that {{async-lease}} is different
{noformat}
19:09:26.346 DEBUG [oak-scheduled-executor-23] ConflictValidator.java:91 Commit
failed due to unresolved conflicts in /:async = {addExistingProperty =
{async-lease = {1434118165914,1434118164643}},changeChangedProperty =
{async-temp = {<ARRAY>,<ARRAY>}}}
org.apache.jackrabbit.oak.api.CommitFailedException: OakState0001: Unresolved
conflicts in /:async
at
org.apache.jackrabbit.oak.plugins.commit.ConflictValidator.failOnMergeConflict(ConflictValidator.java:84)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.commit.ConflictValidator.propertyAdded(ConflictValidator.java:54)
[classes/:na]
at
org.apache.jackrabbit.oak.spi.commit.EditorDiff.propertyAdded(EditorDiff.java:82)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:378)
[classes/:na]
at
org.apache.jackrabbit.oak.spi.commit.EditorDiff.childNodeChanged(EditorDiff.java:148)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:399)
[classes/:na]
at
org.apache.jackrabbit.oak.spi.commit.EditorDiff.process(EditorDiff.java:52)
[classes/:na]
at
org.apache.jackrabbit.oak.spi.commit.EditorHook.processCommit(EditorHook.java:54)
[classes/:na]
at
org.apache.jackrabbit.oak.spi.commit.CompositeHook.processCommit(CompositeHook.java:61)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$InMemory.merge(DocumentNodeStoreBranch.java:543)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.merge0(DocumentNodeStoreBranch.java:223)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.merge(DocumentNodeStoreBranch.java:171)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.document.DocumentRootBuilder.merge(DocumentRootBuilder.java:158)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.merge(DocumentNodeStore.java:1495)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.mergeWithConcurrencyCheck(AsyncIndexUpdate.java:467)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.access$400(AsyncIndexUpdate.java:76)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate$AsyncUpdateCallback.<init>(AsyncIndexUpdate.java:201)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.updateIndex(AsyncIndexUpdate.java:371)
[classes/:na]
at
org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.run(AsyncIndexUpdate.java:320)
[classes/:na]
19:09:26.347 WARN [oak-scheduled-executor-23] AsyncIndexUpdate.java:342
[async] The index update failed
org.apache.jackrabbit.oak.api.CommitFailedException: OakState0001: Unresolved
conflicts in /:async
at
org.apache.jackrabbit.oak.plugins.commit.ConflictValidator.failOnMergeConflict(ConflictValidator.java:84)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.commit.ConflictValidator.propertyAdded(ConflictValidator.java:54)
~[classes/:na]
at
org.apache.jackrabbit.oak.spi.commit.EditorDiff.propertyAdded(EditorDiff.java:82)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:378)
~[classes/:na]
at
org.apache.jackrabbit.oak.spi.commit.EditorDiff.childNodeChanged(EditorDiff.java:148)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:399)
~[classes/:na]
at
org.apache.jackrabbit.oak.spi.commit.EditorDiff.process(EditorDiff.java:52)
~[classes/:na]
at
org.apache.jackrabbit.oak.spi.commit.EditorHook.processCommit(EditorHook.java:54)
~[classes/:na]
at
org.apache.jackrabbit.oak.spi.commit.CompositeHook.processCommit(CompositeHook.java:61)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$InMemory.merge(DocumentNodeStoreBranch.java:543)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.merge0(DocumentNodeStoreBranch.java:223)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.merge(DocumentNodeStoreBranch.java:171)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.document.DocumentRootBuilder.merge(DocumentRootBuilder.java:158)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.merge(DocumentNodeStore.java:1495)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.mergeWithConcurrencyCheck(AsyncIndexUpdate.java:467)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.access$400(AsyncIndexUpdate.java:76)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate$AsyncUpdateCallback.<init>(AsyncIndexUpdate.java:201)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.updateIndex(AsyncIndexUpdate.java:371)
~[classes/:na]
at
org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.run(AsyncIndexUpdate.java:320)
~[classes/:na]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_55]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
[na:1.7.0_55]
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
[na:1.7.0_55]
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
[na:1.7.0_55]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[na:1.7.0_55]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[na:1.7.0_55]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_55]
{noformat}
This is a valid concurrent update conflict and is very much possible in a
cluster and if async job on both cluster node are running. Current logic in
{{AsyncIndexUpdate#mergeWithConcurrencyCheck}} does not check for that and
treats it as a generic commit failure.
[~alex.parvulescu] Should not we also catch CommitFailedException and if it is
of type STATE and code 1 then treat it as CONCURRENT_UPDATE
> Async index fails with OakState0001: Unresolved conflicts in /:async
> --------------------------------------------------------------------
>
> Key: OAK-2961
> URL: https://issues.apache.org/jira/browse/OAK-2961
> Project: Jackrabbit Oak
> Issue Type: Bug
> Components: mongomk, query
> Affects Versions: 1.2.2, 1.0.14, 1.4
> Reporter: Davide Giannella
> Assignee: Davide Giannella
> Fix For: 1.3.1
>
>
> When running on a mongo cluster, in case of high activity the async index
> could fail with the following exception.
> Reproduced with LucenePropertyIndex.
> {noformat}
> 10:55:13.069 [oak-scheduled-executor-5] WARN
> o.a.j.o.p.index.AsyncIndexUpdate - [async] The index update failed
> org.apache.jackrabbit.oak.api.CommitFailedException: OakState0001: Unresolved
> conflicts in /:async
> at
> org.apache.jackrabbit.oak.plugins.commit.ConflictValidator.failOnMergeConflict(ConflictValidator.java:84)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.commit.ConflictValidator.propertyAdded(ConflictValidator.java:54)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.spi.commit.EditorDiff.propertyAdded(EditorDiff.java:82)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:378)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.spi.commit.EditorDiff.childNodeChanged(EditorDiff.java:148)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState.compareAgainstBaseState(ModifiedNodeState.java:400)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.spi.commit.EditorDiff.process(EditorDiff.java:52)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.spi.commit.EditorHook.processCommit(EditorHook.java:55)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.spi.commit.CompositeHook.processCommit(CompositeHook.java:61)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$InMemory.merge(DocumentNodeStoreBranch.java:528)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.merge0(DocumentNodeStoreBranch.java:219)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.merge(DocumentNodeStoreBranch.java:171)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.document.DocumentRootBuilder.merge(DocumentRootBuilder.java:158)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.merge(DocumentNodeStore.java:1466)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.mergeWithConcurrencyCheck(AsyncIndexUpdate.java:467)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.access$4(AsyncIndexUpdate.java:445)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate$AsyncUpdateCallback.<init>(AsyncIndexUpdate.java:201)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.updateIndex(AsyncIndexUpdate.java:372)
> ~[classes/:na]
> at
> org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.run(AsyncIndexUpdate.java:320)
> ~[classes/:na]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> [na:1.7.0_60]
> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
> [na:1.7.0_60]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
> [na:1.7.0_60]
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
> [na:1.7.0_60]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> [na:1.7.0_60]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> [na:1.7.0_60]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_60]
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
