The issue happen because the clients are removed in both active and standby
node when getting
NCSMDS_DOWN event. In standby node, ntfd get NCSMDS_DOWN event is slower than
next initialize request.
This cause the ntfd will removed all client from data base including new client
of next initialze.
Any action relate to this client will fail.
The fixing is that when getting NCSMDS_DOWN event, ntfd remove client in active
node but does not remove
client in standby node. At standby node, ntfd will remove client when process
the checkpoint of NCSMDS_DOWN event.
---
src/ntf/ntfd/NtfAdmin.cc | 89 +++++++++++++++++++++++++++++++++++++++++++++++
src/ntf/ntfd/NtfAdmin.h | 11 ++++--
src/ntf/ntfd/ntfs_com.c | 15 ++++++++
src/ntf/ntfd/ntfs_com.h | 4 +++
src/ntf/ntfd/ntfs_evt.c | 17 +++++++--
src/ntf/ntfd/ntfs_mbcsv.c | 39 +++++++++++++++++++--
6 files changed, 169 insertions(+), 6 deletions(-)
diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc
index dad00383d..aa616d7ee 100644
--- a/src/ntf/ntfd/NtfAdmin.cc
+++ b/src/ntf/ntfd/NtfAdmin.cc
@@ -467,6 +467,80 @@ void NtfAdmin::clientRemoveMDS(MDS_DEST mds_dest) {
}
/**
+ * Checking if the ntf agent with MDS_DEST is valid
+ *
+ * @param agent_dest
+ */
+bool NtfAdmin::is_valid_ntf_agent(MDS_DEST agent_dest) {
+ TRACE_ENTER();
+ ClientMap::iterator it;
+ bool valid = false;
+ for (it = clientMap.begin(); it != clientMap.end(); it++) {
+ NtfClient *client = it->second;
+ if (client->getMdsDest() == agent_dest) {
+ valid = true;
+ break;
+ }
+ }
+ TRACE_LEAVE2("The validation of ntfa: %d", valid);
+ return valid;
+}
+
+/**
+ * Add the ntfa down to the list. This is helpful to remember the
+ * list of ntfa to process in case failover.
+ *
+ * @param agent_dest
+ */
+void NtfAdmin::AddNtfAgentDown(MDS_DEST agent_dest) {
+ TRACE_ENTER2(" Add ntfa down (%ld) to the list", agent_dest);
+
+ if (is_valid_ntf_agent(agent_dest)) {
+ MDS_DEST *mds_dest = new MDS_DEST;
+ *mds_dest = agent_dest;
+ ntfa_down_list.push_back(mds_dest);
+ }
+}
+
+/**
+ * Remove the ntfa down from the list
+ *
+ * @param agent_dest
+ */
+void NtfAdmin::RemoveNtfAgentDownFromList(MDS_DEST agent_dest) {
+ TRACE_ENTER();
+ std::list<MDS_DEST *>::iterator it;
+ for (it = ntfa_down_list.begin(); it != ntfa_down_list.end(); ++it) {
+ MDS_DEST *mds_dest = *it;
+ if (*mds_dest == agent_dest) {
+ ntfa_down_list.erase(it);
+ TRACE(" Remove ntfa down (%ld) from the list", agent_dest);
+ delete mds_dest;
+ return;
+ }
+ }
+}
+
+/**
+ * Process to clear all ntfa down records in ntfd. The old client of
+ * agent down is removed from database.
+ *
+ * @param agent_dest
+ */
+void NtfAdmin::ProcessNtfAgentDownList() {
+ TRACE_ENTER();
+ std::list<MDS_DEST *>::iterator it = ntfa_down_list.begin();;
+ while(it != ntfa_down_list.end()) {
+ MDS_DEST *mds_dest = *it;
+ it = ntfa_down_list.erase(it);
+ clientRemoveMDS(*mds_dest);
+ delete mds_dest;
+ }
+ TRACE_LEAVE();
+ return;
+}
+
+/**
* The node object where the client who had the subscription is notified
* so it can delete the appropriate subscription and filter object.
*
@@ -992,6 +1066,21 @@ void clientRemoveMDS(MDS_DEST mds_dest) {
NtfAdmin::theNtfAdmin->clientRemoveMDS(mds_dest);
}
+void AddNtfAgentDown(MDS_DEST agent_dest) {
+ osafassert(NtfAdmin::theNtfAdmin != NULL);
+ NtfAdmin::theNtfAdmin->AddNtfAgentDown(agent_dest);
+}
+
+void RemoveNtfAgentDownFromList(MDS_DEST agent_dest) {
+ osafassert(NtfAdmin::theNtfAdmin != NULL);
+ NtfAdmin::theNtfAdmin->RemoveNtfAgentDownFromList(agent_dest);
+}
+
+void ProcessNtfAgentDownList() {
+ osafassert(NtfAdmin::theNtfAdmin != NULL);
+ NtfAdmin::theNtfAdmin->ProcessNtfAgentDownList();
+}
+
void subscriptionRemoved(unsigned int clientId,
SaNtfSubscriptionIdT subscriptionId,
MDS_SYNC_SND_CTXT *mdsCtxt) {
diff --git a/src/ntf/ntfd/NtfAdmin.h b/src/ntf/ntfd/NtfAdmin.h
index 1d51b3c52..5ca899711 100644
--- a/src/ntf/ntfd/NtfAdmin.h
+++ b/src/ntf/ntfd/NtfAdmin.h
@@ -102,6 +102,10 @@ class NtfAdmin {
SaClmClusterChangesT cluster_change, NODE_ID node_id);
bool is_stale_client(unsigned int clientId);
+ void AddNtfAgentDown(MDS_DEST agent_dest);
+ void RemoveNtfAgentDownFromList(MDS_DEST agent_dest);
+ void ProcessNtfAgentDownList();
+
private:
void processNotification(unsigned int clientId,
SaNtfNotificationTypeT notificationType,
@@ -110,14 +114,17 @@ class NtfAdmin {
SaNtfIdentifierT notificationId);
void updateNotIdCounter(SaNtfIdentifierT notification);
+ bool is_valid_ntf_agent(MDS_DEST agent_dest);
typedef std::map<unsigned int, NtfClient *> ClientMap;
ClientMap clientMap;
NotificationMap notificationMap;
SaNtfIdentifierT notificationIdCounter;
unsigned int clientIdCounter;
- std::list<NODE_ID *>
- member_node_list; /*To maintain NCS node_ids of CLM memeber nodes.*/
+ // To maintain NCS node_ids of CLM memeber nodes
+ std::list<NODE_ID *> member_node_list;
+ // The list of ntfa down to help remove the clients in case failover
+ std::list<MDS_DEST *> ntfa_down_list;
};
#endif // NTF_NTFD_NTFADMIN_H_
diff --git a/src/ntf/ntfd/ntfs_com.c b/src/ntf/ntfd/ntfs_com.c
index 5881296b0..5ec585b9a 100644
--- a/src/ntf/ntfd/ntfs_com.c
+++ b/src/ntf/ntfd/ntfs_com.c
@@ -555,6 +555,21 @@ void sendNotConfirmUpdate(unsigned int clientId,
TRACE_LEAVE();
}
+// Checkpoint the ntfa down event
+void sendNtfaDownUpdate(MDS_DEST mdsDest)
+{
+ ntfsv_ckpt_msg_t ckpt;
+ TRACE_ENTER2("mdsDest: %ld", mdsDest);
+
+ memset(&ckpt, 0, sizeof(ckpt));
+ ckpt.header.ckpt_rec_type = NTFS_CKPT_AGENT_DOWN;
+ ckpt.header.num_ckpt_records = 1;
+ ckpt.header.data_len = 1;
+ ckpt.ckpt_rec.agent_dest = mdsDest;
+ update_standby(&ckpt, NCS_MBCSV_ACT_ADD);
+ TRACE_LEAVE();
+}
+
/**
* @brief Send Membership status of node to a lib on that node.
*
diff --git a/src/ntf/ntfd/ntfs_com.h b/src/ntf/ntfd/ntfs_com.h
index a8ae26c9b..f05b6ba6f 100644
--- a/src/ntf/ntfd/ntfs_com.h
+++ b/src/ntf/ntfd/ntfs_com.h
@@ -74,6 +74,9 @@ void notificationSentConfirmed(unsigned int clientId,
void notificationLoggedConfirmed(SaNtfIdentifierT notificationId);
void clientRemoved(unsigned int clientId);
void clientRemoveMDS(MDS_DEST mds_dest);
+void AddNtfAgentDown(MDS_DEST agent_dest);
+void RemoveNtfAgentDownFromList(MDS_DEST agent_dest);
+void ProcessNtfAgentDownList();
void subscriptionRemoved(unsigned int clientId,
SaNtfSubscriptionIdT subscriptionId,
MDS_SYNC_SND_CTXT *mdsCtxt);
@@ -170,6 +173,7 @@ void sendNotConfirmUpdate(unsigned int clientId,
int sendNoOfNotifications(uint32_t num_rec, NCS_UBAID *uba);
int sendNoOfSubscriptions(uint32_t num_rec, NCS_UBAID *uba);
int sendNoOfClients(uint32_t num_rec, NCS_UBAID *uba);
+void sendNtfaDownUpdate(MDS_DEST mdsDest);
/* Calls from c --> c++ layer */
void logEvent();
diff --git a/src/ntf/ntfd/ntfs_evt.c b/src/ntf/ntfd/ntfs_evt.c
index 612c124d6..b709a7c1c 100644
--- a/src/ntf/ntfd/ntfs_evt.c
+++ b/src/ntf/ntfd/ntfs_evt.c
@@ -104,8 +104,18 @@ static uint32_t proc_ntfa_updn_mds_msg(ntfsv_ntfs_evt_t
*evt)
case NTFSV_NTFS_EVT_NTFA_UP:
break;
case NTFSV_NTFS_EVT_NTFA_DOWN:
- /* Remove this NTFA entry from our processing lists */
- clientRemoveMDS(evt->fr_dest);
+ // Remove this NTFA entry from our processing lists
+ if ((ntfs_cb->ha_state == SA_AMF_HA_ACTIVE) ||
+ (ntfs_cb->ha_state == SA_AMF_HA_QUIESCED)) {
+ clientRemoveMDS(evt->fr_dest);
+
+ // Checkpoint ntfa down to standby node
+ if (ntfs_cb->ha_state == SA_AMF_HA_ACTIVE) {
+ sendNtfaDownUpdate(evt->fr_dest);
+ }
+ } else if (ntfs_cb->ha_state == SA_AMF_HA_STANDBY) {
+ AddNtfAgentDown(evt->fr_dest);
+ }
break;
default:
TRACE("Unknown evt type!!!");
@@ -194,6 +204,9 @@ static uint32_t proc_rda_cb_msg(ntfsv_ntfs_evt_t *evt)
* not logged */
checkNotificationList();
}
+
+ // Clear all clients that match in list of ntfa down
+ ProcessNtfAgentDownList();
}
rc = NCSCC_RC_SUCCESS;
diff --git a/src/ntf/ntfd/ntfs_mbcsv.c b/src/ntf/ntfd/ntfs_mbcsv.c
index 3a73ec177..0794e99e0 100644
--- a/src/ntf/ntfd/ntfs_mbcsv.c
+++ b/src/ntf/ntfd/ntfs_mbcsv.c
@@ -508,6 +508,27 @@ static uint32_t enc_mbcsv_send_confirm_msg(NCS_UBAID *uba,
return NCSCC_RC_SUCCESS;
}
+static uint32_t ntfsv_enc_ntfa_down(NCS_UBAID *uba, MDS_DEST *mds_dest)
+{
+ uint8_t *p8;
+
+ TRACE_ENTER();
+ osafassert(uba != NULL);
+
+ /** encode the contents **/
+ p8 = ncs_enc_reserve_space(uba, 8);
+ if (!p8) {
+ TRACE("ncs_enc_reserve_space failed");
+ return NCSCC_RC_OUT_OF_MEM;
+ }
+
+ ncs_encode_64bit(&p8, *mds_dest);
+ ncs_enc_claim_space(uba, 8);
+
+ TRACE_LEAVE();
+ return NCSCC_RC_SUCCESS;
+}
+
/****************************************************************************
* Name : ckpt_encode_async_update
*
@@ -557,6 +578,7 @@ static uint32_t ckpt_encode_async_update(ntfs_cb_t
*ntfs_cb, EDU_HDL edu_hdl,
ntfs_ckpt_subscribe_t subscribe_rec;
ntfs_ckpt_unsubscribe_t unsubscribe_rec;
EDU_ERR ederror;
+ MDS_DEST mds_dest;
case NTFS_CKPT_INITIALIZE_REC:
@@ -590,6 +612,17 @@ static uint32_t ckpt_encode_async_update(ntfs_cb_t
*ntfs_cb, EDU_HDL edu_hdl,
TRACE_2("eduerr: %x", ederror);
}
break;
+ case NTFS_CKPT_AGENT_DOWN:
+ // Encode RegHeader
+ ckpt_hdr.ckpt_rec_type = NTFS_CKPT_AGENT_DOWN;
+ ckpt_hdr.num_ckpt_records = 1;
+ ckpt_hdr.data_len = 0;
+ enc_ckpt_header(pheader, ckpt_hdr);
+
+ TRACE_2("AGENT DOWN: AUPDATE");
+ mds_dest = data->ckpt_rec.agent_dest;
+ rc = ntfsv_enc_ntfa_down(uba, &mds_dest);
+ break;
case NTFS_CKPT_SUBSCRIBE:
TRACE("Async update NTFS_CKPT_SUBSCRIBE");
ckpt_hdr.ckpt_rec_type = NTFS_CKPT_SUBSCRIBE;
@@ -1389,8 +1422,10 @@ static uint32_t ckpt_proc_finalize_rec(ntfs_cb_t *cb,
ntfsv_ckpt_msg_t *data)
static uint32_t ckpt_proc_agent_down_rec(ntfs_cb_t *cb, ntfsv_ckpt_msg_t *data)
{
TRACE_ENTER();
- /* Remove this NTFA entry */
- clientRemoveMDS(data->ckpt_rec.agent_dest);
+ // Remove this NTFA entry
+ MDS_DEST agent_dest = data->ckpt_rec.agent_dest;
+ RemoveNtfAgentDownFromList(agent_dest);
+ clientRemoveMDS(agent_dest);
TRACE_LEAVE();
return NCSCC_RC_SUCCESS;
}
--
2.13.0
------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot
_______________________________________________
Opensaf-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/opensaf-devel
