[consensus] add queue/cmeta consistency DCHECK

Added an assertion on the consistency of the consensus metadata and
the consensus queue (effective for DEBUG builds only).

This is in the context of tracking and fixing the bug we hit earlier,
and we guess it happened because of the issue described in KUDU-2274.

Change-Id: Id9c7a16add29c8ab0228085c2009ee17a435d3e8
Reviewed-on: http://gerrit.cloudera.org:8080/9272
Tested-by: Kudu Jenkins
Reviewed-by: Mike Percy <mpe...@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/194fd8b1
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/194fd8b1
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/194fd8b1

Branch: refs/heads/master
Commit: 194fd8b169f29aafbd78a47709ac51d2e8354a1a
Parents: ba33f03
Author: Alexey Serbin <aser...@cloudera.com>
Authored: Fri Feb 9 15:25:50 2018 -0800
Committer: Mike Percy <mpe...@apache.org>
Committed: Sat Feb 10 02:06:37 2018 +0000

----------------------------------------------------------------------
 src/kudu/consensus/consensus_queue.cc | 5 +++++
 src/kudu/consensus/consensus_queue.h  | 3 +++
 src/kudu/consensus/raft_consensus.cc  | 9 +++++++++
 3 files changed, 17 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/194fd8b1/src/kudu/consensus/consensus_queue.cc
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/consensus_queue.cc 
b/src/kudu/consensus/consensus_queue.cc
index c53ad1d..4f81b88 100644
--- a/src/kudu/consensus/consensus_queue.cc
+++ b/src/kudu/consensus/consensus_queue.cc
@@ -1143,6 +1143,11 @@ bool PeerMessageQueue::IsCommittedIndexInCurrentTerm() 
const {
       queue_state_.committed_index >= 
*queue_state_.first_index_in_current_term;
 }
 
+bool PeerMessageQueue::IsInLeaderMode() const {
+  std::lock_guard<simple_spinlock> lock(queue_lock_);
+  return queue_state_.mode == Mode::LEADER;
+}
+
 int64_t PeerMessageQueue::GetMajorityReplicatedIndexForTests() const {
   std::lock_guard<simple_spinlock> lock(queue_lock_);
   return queue_state_.majority_replicated_index;

http://git-wip-us.apache.org/repos/asf/kudu/blob/194fd8b1/src/kudu/consensus/consensus_queue.h
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/consensus_queue.h 
b/src/kudu/consensus/consensus_queue.h
index c64ae86..e3a148c 100644
--- a/src/kudu/consensus/consensus_queue.h
+++ b/src/kudu/consensus/consensus_queue.h
@@ -328,6 +328,9 @@ class PeerMessageQueue {
   // Return true if the committed index falls within the current term.
   bool IsCommittedIndexInCurrentTerm() const;
 
+  // Whether the queue run in the leader mode.
+  bool IsInLeaderMode() const;
+
   // Returns the current majority replicated index, for tests.
   int64_t GetMajorityReplicatedIndexForTests() const;
 

http://git-wip-us.apache.org/repos/asf/kudu/blob/194fd8b1/src/kudu/consensus/raft_consensus.cc
----------------------------------------------------------------------
diff --git a/src/kudu/consensus/raft_consensus.cc 
b/src/kudu/consensus/raft_consensus.cc
index a699a24..98069d7 100644
--- a/src/kudu/consensus/raft_consensus.cc
+++ b/src/kudu/consensus/raft_consensus.cc
@@ -517,6 +517,8 @@ Status RaftConsensus::StepDown(LeaderStepDownResponsePB* 
resp) {
   TRACE_EVENT0("consensus", "RaftConsensus::StepDown");
   ThreadRestrictions::AssertWaitAllowed();
   LockGuard l(lock_);
+  DCHECK((queue_->IsInLeaderMode() && cmeta_->active_role() == 
RaftPeerPB::LEADER) ||
+         (!queue_->IsInLeaderMode() && cmeta_->active_role() != 
RaftPeerPB::LEADER));
   RETURN_NOT_OK(CheckRunningUnlocked());
   if (cmeta_->active_role() != RaftPeerPB::LEADER) {
     LOG_WITH_PREFIX_UNLOCKED(INFO) << "Rejecting request to step down while 
not leader";
@@ -2772,8 +2774,15 @@ Status RaftConsensus::CheckActiveLeaderUnlocked() const {
   RaftPeerPB::Role role = cmeta_->active_role();
   switch (role) {
     case RaftPeerPB::LEADER:
+      // Check for the consistency of the information in the consensus metadata
+      // and the state of the consensus queue.
+      DCHECK(queue_->IsInLeaderMode());
       return Status::OK();
+
     default:
+      // Check for the consistency of the information in the consensus metadata
+      // and the state of the consensus queue.
+      DCHECK(!queue_->IsInLeaderMode());
       return Status::IllegalState(Substitute("Replica $0 is not leader of this 
config. Role: $1. "
                                              "Consensus state: $2",
                                              peer_uuid(),

Reply via email to