[
https://issues.apache.org/jira/browse/CASSANDRA-9935?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15030436#comment-15030436
]
mlowicki commented on CASSANDRA-9935:
-------------------------------------
Tried to run repair once again after online scrub and cleanup on all nodes.
Failed with the same error. This is what I've found in logs:
{code}
ERROR [ValidationExecutor:1089] 2015-11-28 04:33:15,865 Validator.java:245 -
Failed creating a merkle tree for [repair #0f9c5530-9589-11e5-b036-75bb514ae072
on sync/entity2, (-6842825601551036942,-6841068234348096268]], /10.210.3.221
(see log for details)
ERROR [ValidationExecutor:1089] 2015-11-28 04:33:15,866
CassandraDaemon.java:227 - Exception in thread
Thread[ValidationExecutor:1089,1,main]
java.lang.AssertionError: row DecoratedKey(-6842806631972123001,
00093238333134323933320000040000c3c700) received out of order wrt
DecoratedKey(-6841074726771668561, 00093231363735323034340000040000c3c700)
at org.apache.cassandra.repair.Validator.add(Validator.java:127)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.compaction.CompactionManager.doValidationCompaction(CompactionManager.java:1010)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.compaction.CompactionManager.access$600(CompactionManager.java:94)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.db.compaction.CompactionManager$9.call(CompactionManager.java:622)
~[apache-cassandra-2.1.11.jar:2.1.11]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
~[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
~[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[na:1.7.0_80]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
ERROR [AntiEntropySessions:1957] 2015-11-28 04:33:15,868 RepairSession.java:303
- [repair #0f9c5530-9589-11e5-b036-75bb514ae072] session completed with the
following error
org.apache.cassandra.exceptions.RepairException: [repair
#0f9c5530-9589-11e5-b036-75bb514ae072 on sync/entity2,
(-6842825601551036942,-6841068234348096268]] Validation failed in /10.210.3.221
at
org.apache.cassandra.repair.RepairSession.validationComplete(RepairSession.java:166)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:406)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:134)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[na:1.7.0_80]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
{code}
{code}
ERROR [AntiEntropySessions:1957] 2015-11-28 04:33:15,869
CassandraDaemon.java:227 - Exception in thread
Thread[AntiEntropySessions:1957,5,RMI Runtime]
java.lang.RuntimeException: org.apache.cassandra.exceptions.RepairException:
[repair #0f9c5530-9589-11e5-b036-75bb514ae072 on sync/entity2,
(-6842825601551036942,-6841068234348096268]] Validation failed in /10.210.3.221
at com.google.common.base.Throwables.propagate(Throwables.java:160)
~[guava-16.0.jar:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
~[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
~[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
~[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[na:1.7.0_80]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
Caused by: org.apache.cassandra.exceptions.RepairException: [repair
#0f9c5530-9589-11e5-b036-75bb514ae072 on sync/entity2,
(-6842825601551036942,-6841068234348096268]] Validation failed in /10.210.3.221
at
org.apache.cassandra.repair.RepairSession.validationComplete(RepairSession.java:166)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:406)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:134)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64)
~[apache-cassandra-2.1.11.jar:2.1.11]
... 3 common frames omitted
{code}
{code}
ERROR [Thread-6628] 2015-11-28 08:17:03,350 StorageService.java:2999 - Repair
session 93837260-92fb-11e5-b036-75bb514ae072 for range
(-6012485790753833422,-6009995015166063234] failed with error
org.apache.cassandra.exceptions.RepairException: [repair
#93837260-92fb-11e5-b036-75bb514ae072 on sync/entity2,
(-6012485790753833422,-6009995015166063234]] Validation failed in /10.210.3.118
java.util.concurrent.ExecutionException: java.lang.RuntimeException:
org.apache.cassandra.exceptions.RepairException: [repair
#93837260-92fb-11e5-b036-75bb514ae072 on sync/entity2,
(-6012485790753833422,-6009995015166063234]] Validation failed in /10.210.3.118
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.get(FutureTask.java:188)
[na:1.7.0_80]
at
org.apache.cassandra.service.StorageService$4.runMayThrow(StorageService.java:2990)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
Caused by: java.lang.RuntimeException:
org.apache.cassandra.exceptions.RepairException: [repair
#93837260-92fb-11e5-b036-75bb514ae072 on sync/entity2,
(-6012485790753833422,-6009995015166063234]] Validation failed in /10.210.3.118
at com.google.common.base.Throwables.propagate(Throwables.java:160)
~[guava-16.0.jar:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
~[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
~[na:1.7.0_80]
... 1 common frames omitted
Caused by: org.apache.cassandra.exceptions.RepairException: [repair
#93837260-92fb-11e5-b036-75bb514ae072 on sync/entity2,
(-6012485790753833422,-6009995015166063234]] Validation failed in /10.210.3.118
at
org.apache.cassandra.repair.RepairSession.validationComplete(RepairSession.java:166)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:406)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:134)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64)
~[apache-cassandra-2.1.11.jar:2.1.11]
... 3 common frames omitted
{code}
{code}
ERROR [Thread-6628] 2015-11-28 08:17:03,404 StorageService.java:2999 - Repair
session 89fa2b70-933d-11e5-b036-75bb514ae072 for range
(-5867793819051725444,-5865919628027816979] failed with error
org.apache.cassandra.exceptions.RepairException: [repair
#89fa2b70-933d-11e5-b036-75bb514ae072 on sync/entity_by_id2,
(-5867793819051725444,-5865919628027816979]] Validation failed in /10.210.3.117
java.util.concurrent.ExecutionException: java.lang.RuntimeException:
org.apache.cassandra.exceptions.RepairException: [repair
#89fa2b70-933d-11e5-b036-75bb514ae072 on sync/entity_by_id2,
(-5867793819051725444,-5865919628027816979]] Validation failed in /10.210.3.117
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.get(FutureTask.java:188)
[na:1.7.0_80]
at
org.apache.cassandra.service.StorageService$4.runMayThrow(StorageService.java:2990)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
Caused by: java.lang.RuntimeException:
org.apache.cassandra.exceptions.RepairException: [repair
#89fa2b70-933d-11e5-b036-75bb514ae072 on sync/entity_by_id2,
(-5867793819051725444,-5865919628027816979]] Validation failed in /10.210.3.117
at com.google.common.base.Throwables.propagate(Throwables.java:160)
~[guava-16.0.jar:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
~[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
~[na:1.7.0_80]
... 1 common frames omitted
Caused by: org.apache.cassandra.exceptions.RepairException: [repair
#89fa2b70-933d-11e5-b036-75bb514ae072 on sync/entity_by_id2,
(-5867793819051725444,-5865919628027816979]] Validation failed in /10.210.3.117
at
org.apache.cassandra.repair.RepairSession.validationComplete(RepairSession.java:166)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:406)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:134)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64)
~[apache-cassandra-2.1.11.jar:2.1.11]
... 3 common frames omitted
{code}
{code}
ERROR [Thread-6628] 2015-11-28 08:17:03,446 StorageService.java:2999 - Repair
session 3ff36a20-9372-11e5-b036-75bb514ae072 for range
(8066543735336862962,8074446636728465478] failed with error
java.io.IOException: Endpoint /10.210.3.230 died
java.util.concurrent.ExecutionException: java.lang.RuntimeException:
java.io.IOException: Endpoint /10.210.3.230 died
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.get(FutureTask.java:188)
[na:1.7.0_80]
at
org.apache.cassandra.service.StorageService$4.runMayThrow(StorageService.java:2990)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
Caused by: java.lang.RuntimeException: java.io.IOException: Endpoint
/10.210.3.230 died
at com.google.common.base.Throwables.propagate(Throwables.java:160)
~[guava-16.0.jar:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
~[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
~[na:1.7.0_80]
... 1 common frames omitted
Caused by: java.io.IOException: Endpoint /10.210.3.230 died
at
org.apache.cassandra.repair.RepairSession.failedNode(RepairSession.java:351)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.repair.RepairSession.convict(RepairSession.java:386)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.gms.FailureDetector.interpret(FailureDetector.java:276)
~[apache-cassandra-2.1.11.jar:2.1.11]
at org.apache.cassandra.gms.Gossiper.doStatusCheck(Gossiper.java:758)
~[apache-cassandra-2.1.11.jar:2.1.11]
at org.apache.cassandra.gms.Gossiper.access$800(Gossiper.java:66)
~[apache-cassandra-2.1.11.jar:2.1.11]
at org.apache.cassandra.gms.Gossiper$GossipTask.run(Gossiper.java:180)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.concurrent.DebuggableScheduledThreadPoolExecutor$UncomplainingRunnable.run(DebuggableScheduledThreadPoolExecutor.java:118)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
[na:1.7.0_80]
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
~[na:1.7.0_80]
at
java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
~[na:1.7.0_80]
... 3 common frames omitted
{code}
{code}
ERROR [Thread-6628] 2015-11-28 08:17:03,555 StorageService.java:2999 - Repair
session 72d57040-943b-11e5-b036-75bb514ae072 for range
(-2928915626059257529,-2921716383005026147] failed with error
org.apache.cassandra.exceptions.RepairException: [repair
#72d57040-943b-11e5-b036-75bb514ae072 on sync/entity2,
(-2928915626059257529,-2921716383005026147]] Validation failed in /10.210.3.221
java.util.concurrent.ExecutionException: java.lang.RuntimeException:
org.apache.cassandra.exceptions.RepairException: [repair
#72d57040-943b-11e5-b036-75bb514ae072 on sync/entity2,
(-2928915626059257529,-2921716383005026147]] Validation failed in /10.210.3.221
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.get(FutureTask.java:188)
[na:1.7.0_80]
at
org.apache.cassandra.service.StorageService$4.runMayThrow(StorageService.java:2990)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
Caused by: java.lang.RuntimeException:
org.apache.cassandra.exceptions.RepairException: [repair
#72d57040-943b-11e5-b036-75bb514ae072 on sync/entity2,
(-2928915626059257529,-2921716383005026147]] Validation failed in /10.210.3.221
at com.google.common.base.Throwables.propagate(Throwables.java:160)
~[guava-16.0.jar:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
~[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
~[na:1.7.0_80]
... 1 common frames omitted
Caused by: org.apache.cassandra.exceptions.RepairException: [repair
#72d57040-943b-11e5-b036-75bb514ae072 on sync/entity2,
(-2928915626059257529,-2921716383005026147]] Validation failed in /10.210.3.221
at
org.apache.cassandra.repair.RepairSession.validationComplete(RepairSession.java:166)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:406)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:134)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64)
~[apache-cassandra-2.1.11.jar:2.1.11]
... 3 common frames omitted
{code}
{code}
ERROR [Thread-6628] 2015-11-28 08:17:03,687 StorageService.java:2999 - Repair
session e9f771a0-94fe-11e5-b036-75bb514ae072 for range
(-2890799998431679809,-2889623741271856504] failed with error
org.apache.cassandra.exceptions.RepairException: [repair
#e9f771a0-94fe-11e5-b036-75bb514ae072 on sync/entity2,
(-2890799998431679809,-2889623741271856504]] Validation failed in /10.210.3.221
java.util.concurrent.ExecutionException: java.lang.RuntimeException:
org.apache.cassandra.exceptions.RepairException: [repair
#e9f771a0-94fe-11e5-b036-75bb514ae072 on sync/entity2,
(-2890799998431679809,-2889623741271856504]] Validation failed in /10.210.3.221
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.get(FutureTask.java:188)
[na:1.7.0_80]
at
org.apache.cassandra.service.StorageService$4.runMayThrow(StorageService.java:2990)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
Caused by: java.lang.RuntimeException:
org.apache.cassandra.exceptions.RepairException: [repair
#e9f771a0-94fe-11e5-b036-75bb514ae072 on sync/entity2,
(-2890799998431679809,-2889623741271856504]] Validation failed in /10.210.3.221
at com.google.common.base.Throwables.propagate(Throwables.java:160)
~[guava-16.0.jar:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
~[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
~[na:1.7.0_80]
... 1 common frames omitted
Caused by: org.apache.cassandra.exceptions.RepairException: [repair
#e9f771a0-94fe-11e5-b036-75bb514ae072 on sync/entity2,
(-2890799998431679809,-2889623741271856504]] Validation failed in /10.210.3.221
at
org.apache.cassandra.repair.RepairSession.validationComplete(RepairSession.java:166)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:406)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:134)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64)
~[apache-cassandra-2.1.11.jar:2.1.11]
... 3 common frames omitted
{code}
{code}
ERROR [Thread-6628] 2015-11-28 08:17:03,766 StorageService.java:2999 - Repair
session 0f9c5530-9589-11e5-b036-75bb514ae072 for range
(-6842825601551036942,-6841068234348096268] failed with error
org.apache.cassandra.exceptions.RepairException: [repair
#0f9c5530-9589-11e5-b036-75bb514ae072 on sync/entity2,
(-6842825601551036942,-6841068234348096268]] Validation failed in /10.210.3.221
java.util.concurrent.ExecutionException: java.lang.RuntimeException:
org.apache.cassandra.exceptions.RepairException: [repair
#0f9c5530-9589-11e5-b036-75bb514ae072 on sync/entity2,
(-6842825601551036942,-6841068234348096268]] Validation failed in /10.210.3.221
at java.util.concurrent.FutureTask.report(FutureTask.java:122)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.get(FutureTask.java:188)
[na:1.7.0_80]
at
org.apache.cassandra.service.StorageService$4.runMayThrow(StorageService.java:2990)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
Caused by: java.lang.RuntimeException:
org.apache.cassandra.exceptions.RepairException: [repair
#0f9c5530-9589-11e5-b036-75bb514ae072 on sync/entity2,
(-6842825601551036942,-6841068234348096268]] Validation failed in /10.210.3.221
at com.google.common.base.Throwables.propagate(Throwables.java:160)
~[guava-16.0.jar:na]
at
org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)
[apache-cassandra-2.1.11.jar:2.1.11]
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
[na:1.7.0_80]
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
~[na:1.7.0_80]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
~[na:1.7.0_80]
... 1 common frames omitted
Caused by: org.apache.cassandra.exceptions.RepairException: [repair
#0f9c5530-9589-11e5-b036-75bb514ae072 on sync/entity2,
(-6842825601551036942,-6841068234348096268]] Validation failed in /10.210.3.221
at
org.apache.cassandra.repair.RepairSession.validationComplete(RepairSession.java:166)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:406)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:134)
~[apache-cassandra-2.1.11.jar:2.1.11]
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:64)
~[apache-cassandra-2.1.11.jar:2.1.11]
... 3 common frames omitted
{code}
Repair failed between 08:00 and 08:40.
{code}
ERROR [Thread-6628] 2015-11-28 08:17:03,446 StorageService.java:2999 - Repair
session 3ff36a20-9372-11e5-b036-75bb514ae072 for range
(8066543735336862962,8074446636728465478] failed with error
java.io.IOException: Endpoint /10.210.3.230 died
{code}
Is interesting but I haven't found more about it in logs (Attached logs from
box where I've started repair - 10.210.3.221 and 10.210.3.230).
If started repair for one of failed ranges then it works fine:
{code}
root@db1:~# time nodetool repair --in-local-dc -st -2890799998431679809 -et
-2889623741271856504 sync entity2
[2015-11-28 08:36:41,736] Starting repair command #5, repairing 1 ranges for
keyspace sync (parallelism=SEQUENTIAL, full=true)
[2015-11-28 08:37:48,286] Repair session 26483200-95ab-11e5-b036-75bb514ae072
for range (-2890799998431679809,-2889623741271856504] finished
[2015-11-28 08:37:48,286] Repair command #5 finished
real 1m8.393s
user 0m2.620s
sys 0m0.184s
{code}
> Repair fails with RuntimeException
> ----------------------------------
>
> Key: CASSANDRA-9935
> URL: https://issues.apache.org/jira/browse/CASSANDRA-9935
> Project: Cassandra
> Issue Type: Bug
> Environment: C* 2.1.8, Debian Wheezy
> Reporter: mlowicki
> Assignee: Yuki Morishita
> Fix For: 2.1.x
>
> Attachments: db1.sync.lati.osa.cassandra.log,
> db5.sync.lati.osa.cassandra.log, system.log.10.210.3.117,
> system.log.10.210.3.221, system.log.10.210.3.230
>
>
> We had problems with slow repair in 2.1.7 (CASSANDRA-9702) but after upgrade
> to 2.1.8 it started to work faster but now it fails with:
> {code}
> ...
> [2015-07-29 20:44:03,956] Repair session 23a811b0-3632-11e5-a93e-4963524a8bde
> for range (-5474076923322749342,-5468600594078911162] finished
> [2015-07-29 20:44:03,957] Repair session 336f8740-3632-11e5-a93e-4963524a8bde
> for range (-8631877858109464676,-8624040066373718932] finished
> [2015-07-29 20:44:03,957] Repair session 4ccd8430-3632-11e5-a93e-4963524a8bde
> for range (-5372806541854279315,-5369354119480076785] finished
> [2015-07-29 20:44:03,957] Repair session 59f129f0-3632-11e5-a93e-4963524a8bde
> for range (8166489034383821955,8168408930184216281] finished
> [2015-07-29 20:44:03,957] Repair session 6ae7a9a0-3632-11e5-a93e-4963524a8bde
> for range (6084602890817326921,6088328703025510057] finished
> [2015-07-29 20:44:03,957] Repair session 8938e4a0-3632-11e5-a93e-4963524a8bde
> for range (-781874602493000830,-781745173070807746] finished
> [2015-07-29 20:44:03,957] Repair command #4 finished
> error: nodetool failed, check server logs
> -- StackTrace --
> java.lang.RuntimeException: nodetool failed, check server logs
> at
> org.apache.cassandra.tools.NodeTool$NodeToolCmd.run(NodeTool.java:290)
> at org.apache.cassandra.tools.NodeTool.main(NodeTool.java:202)
> {code}
> After running:
> {code}
> nodetool repair --partitioner-range --parallel --in-local-dc sync
> {code}
> Last records in logs regarding repair are:
> {code}
> INFO [Thread-173887] 2015-07-29 20:44:03,956 StorageService.java:2952 -
> Repair session 09ff9e40-3632-11e5-a93e-4963524a8bde for range
> (-7695808664784761779,-7693529816291585568] finished
> INFO [Thread-173887] 2015-07-29 20:44:03,956 StorageService.java:2952 -
> Repair session 17d8d860-3632-11e5-a93e-4963524a8bde for range
> (8063716953988492222,8065203836608925992] finished
> INFO [Thread-173887] 2015-07-29 20:44:03,956 StorageService.java:2952 -
> Repair session 23a811b0-3632-11e5-a93e-4963524a8bde for range
> (-5474076923322749342,-5468600594078911162] finished
> INFO [Thread-173887] 2015-07-29 20:44:03,956 StorageService.java:2952 -
> Repair session 336f8740-3632-11e5-a93e-4963524a8bde for range
> (-8631877858109464676,-8624040066373718932] finished
> INFO [Thread-173887] 2015-07-29 20:44:03,957 StorageService.java:2952 -
> Repair session 4ccd8430-3632-11e5-a93e-4963524a8bde for range
> (-5372806541854279315,-5369354119480076785] finished
> INFO [Thread-173887] 2015-07-29 20:44:03,957 StorageService.java:2952 -
> Repair session 59f129f0-3632-11e5-a93e-4963524a8bde for range
> (8166489034383821955,8168408930184216281] finished
> INFO [Thread-173887] 2015-07-29 20:44:03,957 StorageService.java:2952 -
> Repair session 6ae7a9a0-3632-11e5-a93e-4963524a8bde for range
> (6084602890817326921,6088328703025510057] finished
> INFO [Thread-173887] 2015-07-29 20:44:03,957 StorageService.java:2952 -
> Repair session 8938e4a0-3632-11e5-a93e-4963524a8bde for range
> (-781874602493000830,-781745173070807746] finished
> {code}
> but a bit above I see (at least two times in attached log):
> {code}
> ERROR [Thread-173887] 2015-07-29 20:44:03,853 StorageService.java:2959 -
> Repair session 1b07ea50-3608-11e5-a93e-4963524a8bde for range
> (5765414319217852786,5781018794516851576] failed with error
> org.apache.cassandra.exceptions.RepairException: [repair
> #1b07ea50-3608-11e5-a93e-4963524a8bde on sync/entity_by_id2,
> (5765414319217852786,5781018794516851576]] Validation failed in /10.195.15.162
> java.util.concurrent.ExecutionException: java.lang.RuntimeException:
> org.apache.cassandra.exceptions.RepairException: [repair
> #1b07ea50-3608-11e5-a93e-4963524a8bde on sync/entity_by_id2,
> (5765414319217852786,5781018794516851576]] Validation failed in /10.195.15.162
> at java.util.concurrent.FutureTask.report(FutureTask.java:122)
> [na:1.7.0_80]
> at java.util.concurrent.FutureTask.get(FutureTask.java:188)
> [na:1.7.0_80]
> at
> org.apache.cassandra.service.StorageService$4.runMayThrow(StorageService.java:2950)
> ~[apache-cassandra-2.1.8.jar:2.1.8]
> at
> org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:28)
> [apache-cassandra-2.1.8.jar:2.1.8]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> [na:1.7.0_80]
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> [na:1.7.0_80]
> at java.lang.Thread.run(Thread.java:745) [na:1.7.0_80]
> Caused by: java.lang.RuntimeException:
> org.apache.cassandra.exceptions.RepairException: [repair
> #1b07ea50-3608-11e5-a93e-4963524a8bde on sync/entity_by_id2,
> (5765414319217852786,5781018794516851576]] Validation failed in /10.195.15.162
> at com.google.common.base.Throwables.propagate(Throwables.java:160)
> ~[guava-16.0.jar:na]
> at
> org.apache.cassandra.utils.WrappedRunnable.run(WrappedRunnable.java:32)
> [apache-cassandra-2.1.8.jar:2.1.8]
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> [na:1.7.0_80]
> at java.util.concurrent.FutureTask.run(FutureTask.java:262)
> [na:1.7.0_80]
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
> ~[na:1.7.0_80]
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> ~[na:1.7.0_80] ... 1 common frames omitted
> Caused by: org.apache.cassandra.exceptions.RepairException: [repair
> #1b07ea50-3608-11e5-a93e-4963524a8bde on sync/entity_by_id2,
> (5765414319217852786,5781018794516851576]] Validation failed in /10.195.15.162
> at
> org.apache.cassandra.repair.RepairSession.validationComplete(RepairSession.java:166)
> ~[apache-cassandra-2.1.8.jar:2.1.8] at
> org.apache.cassandra.service.ActiveRepairService.handleMessage(ActiveRepairService.java:406)
> ~[apache-cassandra-2.1.8.jar:2.1.8]
> at
> org.apache.cassandra.repair.RepairMessageVerbHandler.doVerb(RepairMessageVerbHandler.java:134)
> ~[apache-cassandra-2.1.8.jar:2.1.8] at
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:62)
> ~[apache-cassandra-2.1.8.jar:2.1.8]
> ... 3 common frames omittedINFO [Thread-173887] 2015-07-29
> 20:44:03,854 StorageService.java:2952 - Repair session
> 846d9300-3608-11e5-a93e-4963524a8bde for range (-6705935
> 742755245856,-6704072966568763453] finished
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
