Repository: cassandra Updated Branches: refs/heads/trunk 3234c0704 -> af3748909
Improve handling of IR prepare failures Patch by Blake Eggleston; Reviewed by Marcus Eriksson for CASSANDRA-13672 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/af374890 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/af374890 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/af374890 Branch: refs/heads/trunk Commit: af37489092ca90bca336538adad02fb5ba859945 Parents: 3234c07 Author: Blake Eggleston <bdeggles...@gmail.com> Authored: Wed Jul 5 13:28:04 2017 -0700 Committer: Blake Eggleston <bdeggles...@gmail.com> Committed: Thu Jul 6 10:35:53 2017 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../repair/consistent/CoordinatorSession.java | 4 ++++ .../cassandra/repair/consistent/LocalSessions.java | 17 +++++++++++++++-- .../repair/consistent/PendingAntiCompaction.java | 4 +++- .../repair/consistent/LocalSessionTest.java | 2 +- 5 files changed, 24 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/af374890/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 6cd8bc5..9584f63 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.0 + * Improve handling of IR prepare failures (CASSANDRA-13672) * Send IR coordinator messages synchronously (CASSANDRA-13673) * Flush system.repair table before IR finalize promise (CASSANDRA-13660) * Fix column filter creation for wildcard queries (CASSANDRA-13650) http://git-wip-us.apache.org/repos/asf/cassandra/blob/af374890/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java index 830ed2c..d0ec7fd 100644 --- a/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java +++ b/src/java/org/apache/cassandra/repair/consistent/CoordinatorSession.java @@ -240,6 +240,10 @@ public class CoordinatorSession extends ConsistentSession } } setAll(State.FAILED); + + String exceptionMsg = String.format("Incremental repair session %s has failed", sessionID); + finalizeProposeFuture.setException(new RuntimeException(exceptionMsg)); + prepareFuture.setException(new RuntimeException(exceptionMsg)); } private static String formatDuration(long then, long now) http://git-wip-us.apache.org/repos/asf/cassandra/blob/af374890/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java index 61df2b0..a25f65c 100644 --- a/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java +++ b/src/java/org/apache/cassandra/repair/consistent/LocalSessions.java @@ -568,8 +568,21 @@ public class LocalSessions public void onFailure(Throwable t) { - logger.error(String.format("Prepare phase for incremental repair session %s failed", sessionID), t); - failSession(sessionID); + logger.error("Prepare phase for incremental repair session {} failed", sessionID, t); + if (t instanceof PendingAntiCompaction.SSTableAcquisitionException) + { + logger.warn("Prepare phase for incremental repair session {} was unable to " + + "acquire exclusive access to the neccesary sstables. " + + "This is usually caused by running multiple incremental repairs on nodes that share token ranges", + sessionID); + + } + else + { + logger.error("Prepare phase for incremental repair session {} failed", sessionID, t); + } + sendMessage(coordinator, new PrepareConsistentResponse(sessionID, getBroadcastAddress(), false)); + failSession(sessionID, false); executor.shutdown(); } }); http://git-wip-us.apache.org/repos/asf/cassandra/blob/af374890/src/java/org/apache/cassandra/repair/consistent/PendingAntiCompaction.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/repair/consistent/PendingAntiCompaction.java b/src/java/org/apache/cassandra/repair/consistent/PendingAntiCompaction.java index f183d14..e37fb19 100644 --- a/src/java/org/apache/cassandra/repair/consistent/PendingAntiCompaction.java +++ b/src/java/org/apache/cassandra/repair/consistent/PendingAntiCompaction.java @@ -76,6 +76,8 @@ public class PendingAntiCompaction } } + static class SSTableAcquisitionException extends RuntimeException {} + static class AcquisitionCallable implements Callable<AcquireResult> { private final ColumnFamilyStore cfs; @@ -150,7 +152,7 @@ public class PendingAntiCompaction result.abort(); } } - return Futures.immediateFailedFuture(new RuntimeException("unable to acquire sstables")); + return Futures.immediateFailedFuture(new SSTableAcquisitionException()); } else { http://git-wip-us.apache.org/repos/asf/cassandra/blob/af374890/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java index 3b48051..c59462e 100644 --- a/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java +++ b/test/unit/org/apache/cassandra/repair/consistent/LocalSessionTest.java @@ -303,7 +303,7 @@ public class LocalSessionTest extends AbstractRepairTest Assert.assertEquals(session, sessions.loadUnsafe(sessionID)); // ...and we should have sent a success message back to the coordinator - assertMessagesSent(sessions, COORDINATOR, new FailSession(sessionID)); + assertMessagesSent(sessions, COORDINATOR, new PrepareConsistentResponse(sessionID, PARTICIPANT1, false)); } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
