This is an automated email from the ASF dual-hosted git repository. awong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit da55638394d78830b3440b2272f8161afae08ca0 Author: Andrew Wong <[email protected]> AuthorDate: Tue Jan 7 18:07:26 2020 -0800 consensus: throttle warnings when quiescing server When an election fails to start because the server is quiescing, it will log a warning. When there are many replicas on a server, this logging can become very verbose. This throttles such logs to happen at most once per ten seconds. Here's a snippet of the logs from TestQuiescingServerDoesntTriggerElections: Before: W0107 17:55:46.943423 55670 raft_consensus.cc:655] T d1bc343b18e348ada0a4fadb835d8cc1 P f0fd8e85d7a843e6862dfd846ca47351: failed to trigger leader election: Illegal state: leader elections are disabled W0107 17:55:46.947376 55670 raft_consensus.cc:655] T 4fb4aec4dbb94dc9b2af086b761d2632 P f0fd8e85d7a843e6862dfd846ca47351: failed to trigger leader election: Illegal state: leader elections are disabled W0107 17:55:46.949579 55670 raft_consensus.cc:655] T 49bf5837ae0c4d95bf1a176a856c9f37 P f0fd8e85d7a843e6862dfd846ca47351: failed to trigger leader election: Illegal state: leader elections are disabled W0107 17:55:46.992861 55670 raft_consensus.cc:655] T 842d307a87f846efa00bbf0a537fe812 P f0fd8e85d7a843e6862dfd846ca47351: failed to trigger leader election: Illegal state: leader elections are disabled After: W0107 19:16:47.518998 87519 raft_consensus.cc:655] T 1e0aed8879c643d6a8684f6da729a1a3 P 14907cae807b4560b4b8417ba5f24393: failed to trigger leader election: Illegal state: leader elections are disabled (in After, the test didn't last 10 seconds) While it would be nice to retain logs for every replica for grep-ability, I think the improved readability of the logs justifies the loss. Change-Id: I01e3c88c2bd4c53d095b9b5a061439231dcfc9e9 Reviewed-on: http://gerrit.cloudera.org:8080/14989 Reviewed-by: Alexey Serbin <[email protected]> Tested-by: Andrew Wong <[email protected]> --- src/kudu/consensus/raft_consensus.cc | 12 ++++++------ src/kudu/util/logging.h | 7 +++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/kudu/consensus/raft_consensus.cc b/src/kudu/consensus/raft_consensus.cc index 204c79f..8f83ed8 100644 --- a/src/kudu/consensus/raft_consensus.cc +++ b/src/kudu/consensus/raft_consensus.cc @@ -379,8 +379,8 @@ Status RaftConsensus::Start(const ConsensusBootstrapInfo& info, if (IsSingleVoterConfig() && FLAGS_enable_leader_failure_detection) { LOG_WITH_PREFIX(INFO) << "Only one voter in the Raft config. Triggering election immediately"; - WARN_NOT_OK(StartElection(NORMAL_ELECTION, INITIAL_SINGLE_NODE_ELECTION), - "Couldn't start leader election"); + WARN_NOT_OK_EVERY_N_SECS(StartElection(NORMAL_ELECTION, INITIAL_SINGLE_NODE_ELECTION), + "Couldn't start leader election", 10); } // Report become visible to the Master. @@ -650,9 +650,9 @@ void RaftConsensus::ReportFailureDetectedTask() { std::unique_lock<simple_spinlock> try_lock(failure_detector_election_lock_, std::try_to_lock); if (try_lock.owns_lock()) { - WARN_NOT_OK(StartElection(FLAGS_raft_enable_pre_election ? + WARN_NOT_OK_EVERY_N_SECS(StartElection(FLAGS_raft_enable_pre_election ? PRE_ELECTION : NORMAL_ELECTION, ELECTION_TIMEOUT_EXPIRED), - LogPrefixThreadSafe() + "failed to trigger leader election"); + LogPrefixThreadSafe() + "failed to trigger leader election", 10); } } @@ -2698,8 +2698,8 @@ void RaftConsensus::DoElectionCallback(ElectionReason reason, const ElectionResu if (was_pre_election) { // We just won the pre-election. So, we need to call a real election. lock.unlock(); - WARN_NOT_OK(StartElection(NORMAL_ELECTION, reason), - "Couldn't start leader election after successful pre-election"); + WARN_NOT_OK_EVERY_N_SECS(StartElection(NORMAL_ELECTION, reason), + "Couldn't start leader election after successful pre-election", 10); } else { // We won a real election. Convert role to LEADER. SetLeaderUuidUnlocked(peer_uuid()); diff --git a/src/kudu/util/logging.h b/src/kudu/util/logging.h index 87012b1..0d3492a 100644 --- a/src/kudu/util/logging.h +++ b/src/kudu/util/logging.h @@ -164,6 +164,13 @@ class ScopedDisableRedaction { static logging::LogThrottler LOG_THROTTLER; \ KLOG_EVERY_N_SECS_THROTTLER(severity, n_secs, LOG_THROTTLER, "no-tag") +#define WARN_NOT_OK_EVERY_N_SECS(to_call, warning_prefix, n_secs) do { \ + const ::kudu::Status& _s = (to_call); \ + if (PREDICT_FALSE(!_s.ok())) { \ + KLOG_EVERY_N_SECS(WARNING, n_secs) << (warning_prefix) << ": " << _s.ToString() \ + << THROTTLE_MSG; \ + } \ + } while (0) namespace kudu { enum PRIVATE_ThrottleMsg {THROTTLE_MSG};
