Keep the SC with the earlier boot time alive, if split brain is
detected. In the unlikely event that the boot up time
is equal, the node with the lower ID survives.
---
src/rde/rded/rde_cb.h | 2 ++
src/rde/rded/rde_main.cc | 21 ++++++++++++++++++++-
src/rde/rded/rde_mds.cc | 22 +++++++++++++++-------
3 files changed, 37 insertions(+), 8 deletions(-)
diff --git a/src/rde/rded/rde_cb.h b/src/rde/rded/rde_cb.h
index d3f5a24a5..76c8b8edc 100644
--- a/src/rde/rded/rde_cb.h
+++ b/src/rde/rded/rde_cb.h
@@ -43,6 +43,7 @@ struct RDE_CONTROL_BLOCK {
bool monitor_lock_thread_running{false};
bool monitor_takeover_req_thread_running{false};
std::set<NODE_ID> cluster_members{};
+ time_t boot_time;
};
enum RDE_MSG_TYPE {
@@ -59,6 +60,7 @@ enum RDE_MSG_TYPE {
struct rde_peer_info {
PCS_RDA_ROLE ha_role;
+ time_t boot_time; // seconds since epoch
};
struct rde_msg {
diff --git a/src/rde/rded/rde_main.cc b/src/rde/rded/rde_main.cc
index e5813e414..714620485 100644
--- a/src/rde/rded/rde_main.cc
+++ b/src/rde/rded/rde_main.cc
@@ -29,11 +29,13 @@
#include <cerrno>
#include <cstdlib>
#include <cstring>
+#include <ctime>
#include "base/conf.h"
#include "base/daemon.h"
#include "base/logtrace.h"
#include "base/ncs_main_papi.h"
#include "base/osaf_poll.h"
+#include "base/time.h"
#include "mds/mds_papi.h"
#include "nid/agent/nid_api.h"
#include "osaf/consensus/consensus.h"
@@ -207,7 +209,20 @@ static void CheckForSplitBrain(const rde_msg *msg) {
PCS_RDA_ROLE own_role = role->role();
PCS_RDA_ROLE other_role = msg->info.peer_info.ha_role;
if (own_role == PCS_RDA_ACTIVE && other_role == PCS_RDA_ACTIVE) {
- opensaf_reboot(0, nullptr, "Split-brain detected");
+ LOG_EM("Split-brain detected. My node ID 0x%x (%s), peer node ID 0x%x
(%s)",
+ own_node_id, asctime(gmtime(&rde_cb->boot_time)),
+ msg->fr_node_id, asctime(gmtime(&msg->info.peer_info.boot_time)));
+ if (rde_cb->boot_time > msg->info.peer_info.boot_time) {
+ // peer node has an earlier boot up time
+ opensaf_reboot(0, nullptr, "Split-brain detected");
+ } else if (rde_cb->boot_time == msg->info.peer_info.boot_time &&
+ own_node_id > msg->fr_node_id) {
+ // identical boot up time, use node_id to decide
+ opensaf_reboot(0, nullptr, "Split-brain detected");
+ } else {
+ // we have a later boot up time
+ LOG_EM("Peer node will self fence instead of us");
+ }
}
}
@@ -267,6 +282,10 @@ static int initialize_rde() {
goto init_failed;
}
+ struct timespec boot_time;
+ osaf_get_boot_time(&boot_time);
+ rde_cb->boot_time = boot_time.tv_sec;
+
rc = NCSCC_RC_SUCCESS;
init_failed:
diff --git a/src/rde/rded/rde_mds.cc b/src/rde/rded/rde_mds.cc
index 00922eaf8..39862c78a 100644
--- a/src/rde/rded/rde_mds.cc
+++ b/src/rde/rded/rde_mds.cc
@@ -23,16 +23,17 @@
#include "base/ncsencdec_pub.h"
#include "rde/rded/rde_cb.h"
-#define RDE_MDS_PVT_SUBPART_VERSION 1
+#define RDE_MDS_PVT_SUBPART_VERSION 2
static MDS_HDL mds_hdl;
static uint32_t msg_encode(MDS_CALLBACK_ENC_INFO *enc_info) {
+ RDE_CONTROL_BLOCK *rde_cb = rde_get_control_block();
struct rde_msg *msg;
NCS_UBAID *uba;
uint8_t *data;
- enc_info->o_msg_fmt_ver = 1;
+ enc_info->o_msg_fmt_ver = 2;
uba = enc_info->io_uba;
msg = (struct rde_msg *)enc_info->i_msg;
@@ -48,6 +49,12 @@ static uint32_t msg_encode(MDS_CALLBACK_ENC_INFO *enc_info) {
assert(data);
ncs_encode_32bit(&data, msg->info.peer_info.ha_role);
ncs_enc_claim_space(uba, sizeof(uint32_t));
+ if (enc_info->i_rem_svc_pvt_ver >= 2) {
+ data = ncs_enc_reserve_space(uba, sizeof(uint64_t));
+ assert(data);
+ ncs_encode_64bit(&data, rde_cb->boot_time);
+ ncs_enc_claim_space(uba, sizeof(uint64_t));
+ }
break;
default:
@@ -70,11 +77,6 @@ static uint32_t msg_decode(MDS_CALLBACK_DEC_INFO *dec_info) {
goto done;
}
- if (dec_info->i_msg_fmt_ver != 1) {
- rc = NCSCC_RC_FAILURE;
- goto done;
- }
-
msg = static_cast<rde_msg *>(malloc(sizeof(*msg)));
assert(msg);
@@ -94,6 +96,12 @@ static uint32_t msg_decode(MDS_CALLBACK_DEC_INFO *dec_info) {
msg->info.peer_info.ha_role =
static_cast<PCS_RDA_ROLE>(ncs_decode_32bit(&data));
ncs_dec_skip_space(uba, sizeof(uint32_t));
+ if (dec_info->i_msg_fmt_ver >= 2) {
+ data = ncs_dec_flatten_space(uba, data_buff, sizeof(uint64_t));
+ assert(data);
+ msg->info.peer_info.boot_time = ncs_decode_64bit(&data);
+ ncs_dec_skip_space(uba, sizeof(uint64_t));
+ }
break;
default:
--
2.17.1
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel
