[devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Ntfd will not send response to a client when client already down.
This will avoid timeout when ntfd send via mds.
---
src/ntf/ntfd/ntfs_com.c | 120
src/ntf/ntfd/ntfs_com.h | 6 +++
src/ntf/ntfd/ntfs_evt.c | 3 ++
src/ntf/ntfd/ntfs_mds.c | 7 ++-
4 files changed, 135 insertions(+), 1 deletion(-)
diff --git a/src/ntf/ntfd/ntfs_com.c b/src/ntf/ntfd/ntfs_com.c
index 35435bb..c9fce2d 100644
--- a/src/ntf/ntfd/ntfs_com.c
+++ b/src/ntf/ntfd/ntfs_com.c
@@ -29,13 +29,133 @@
#include "ntf/common/ntfsv_enc_dec.h"
#include "ntf/common/ntfsv_mem.h"
#include "ntfs_mbcsv.h"
+#include "base/osaf_utility.h"
#if DISCARDED_TEST
/* TODO REMOVE TEST */
int disc_test_cntr = 1;
#endif
+typedef struct client_down_list_tag {
+ MDS_DEST mds_dest;
+ struct client_down_list_tag *next;
+} CLIENT_DOWN_LIST;
+
+CLIENT_DOWN_LIST *client_down_list_head; /* Client down reccords */
+pthread_mutex_t client_down_mutex;
+bool client_finalized_flag;
+
int activeController() { return (ntfs_cb->ha_state == SA_AMF_HA_ACTIVE); }
+void set_client_finalized_flag(void) { client_finalized_flag = true; }
+
+void mutex_client_down_init(void)
+{
+ int rc;
+
+ TRACE_ENTER();
+ rc = pthread_mutex_init(_down_mutex, NULL);
+ if (rc != 0)
+ osaf_abort(rc);
+ client_finalized_flag = false;
+ TRACE_LEAVE();
+}
+
+/**
+ * @brief Add mds_dest tag into client down list
+ * @param mds_dest
+ */
+void add_client_down_list(MDS_DEST mds_dest)
+{
+ TRACE_ENTER();
+ // Prevent add to down list when client has successfully initialized.
+ if (!client_finalized_flag) {
+ CLIENT_DOWN_LIST *client_down_rec = NULL;
+
+ client_down_rec =
+ (CLIENT_DOWN_LIST *)(malloc(sizeof(CLIENT_DOWN_LIST)));
+ if (client_down_rec == NULL) {
+ LOG_ER("memory allocation for the CLIENT_DOWN_LIST "
+ "failed");
+ return;
+ }
+ memset(client_down_rec, 0, sizeof(CLIENT_DOWN_LIST));
+ client_down_rec->mds_dest = mds_dest;
+ client_down_rec->next = NULL;
+ osaf_mutex_lock_ordie(_down_mutex);
+ if (client_down_list_head == NULL) {
+ client_down_list_head = client_down_rec;
+ } else {
+ CLIENT_DOWN_LIST *p = client_down_list_head;
+
+ while (p->next != NULL)
+ p = p->next;
+ p->next = client_down_rec;
+ }
+ osaf_mutex_unlock_ordie(_down_mutex);
+ TRACE_1("Added MDS dest: %" PRIx64, client_down_rec->mds_dest);
+ }
+ client_finalized_flag = false;
+ TRACE_LEAVE();
+}
+
+/**
+ * @brief Find and remove client from client down list
+ * @param mds_dest
+ */
+void remove_client_down_list(MDS_DEST mds_dest)
+{
+ CLIENT_DOWN_LIST *client_down_rec = client_down_list_head;
+ CLIENT_DOWN_LIST *prev = NULL;
+ TRACE_ENTER();
+ osaf_mutex_lock_ordie(_down_mutex);
+ while (client_down_rec != NULL) {
+ if (mds_dest == client_down_rec->mds_dest) {
+ if (client_down_rec == client_down_list_head) {
+ if (client_down_rec->next == NULL) {
+ client_down_list_head = NULL;
+ } else {
+ client_down_list_head =
+ client_down_rec->next;
+ }
+ } else if (prev) {
+ prev->next = client_down_rec->next;
+ }
+ TRACE("Deleted MDS dest: %" PRIx64,
+ client_down_rec->mds_dest);
+ free(client_down_rec);
+ client_down_rec = NULL;
+ break;
+ }
+ prev = client_down_rec;
+ client_down_rec = client_down_rec->next;
+ }
+ osaf_mutex_unlock_ordie(_down_mutex);
+ TRACE_LEAVE();
+}
+
+/**
+ * @brief Check if client exists in down list
+ * @param mds_dest
+ * @return true/false
+ */
+bool is_in_down_list(MDS_DEST mds_dest)
+{
+ bool found = false;
+ CLIENT_DOWN_LIST *client_down_rec = client_down_list_head;
+ TRACE_ENTER();
+ osaf_mutex_lock_ordie(_down_mutex);
+ while (client_down_rec != NULL) {
+ if (mds_dest == client_down_rec->mds_dest) {
+ found = true;
+ break;
+ }
+ client_down_rec = client_down_rec->next;
+ }
+ osaf_mutex_unlock_ordie(_down_mutex);
+ TRACE_LEAVE();
+ return found;
+}
+
void client_added_res_lib(SaAisErrorT error, unsigned
Re: [devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Hi Minh,
Thanks for your comment.
In case, agent down before clientAdded() then SearchAndSetClientsDownFlag()
does not work.
Best Regards,
ThienHuynh
-Original Message-
From: Minh Hon Chau
Sent: Thursday, October 10, 2019 1:50 AM
To: Thien Minh Huynh ; 'Nguyen Minh Vu'
; thuan.t...@dektech.com.au
Cc: opensaf-devel@lists.sourceforge.net
Subject: Re: [PATCH 1/1] ntfd: Do not send response to client if client down
[#3084]
Hi all,
What I guess from the ticket that it is a race condition between the mds thread
and main thread in ntfd. We normally get NCSDOWN callback from mds, and send
event to main thread to remove the client. But the mds callback here comes in
the middle of processing Initialize().
We have something similar done in ntfd with SearchAndSetClientsDownFlag(),
GetClientDownFlag(), SetClientDownFlag(), can we try to reuse them?
Thanks,
Minh
On 9/10/19 5:10 pm, Thien Minh Huynh wrote:
> Hi Vu,
>
> Thanks for your time to review the patch.
>
> Best Regards,
> ThienHuynh
>
> -Original Message-
> From: Nguyen Minh Vu
> Sent: Wednesday, October 9, 2019 11:15 AM
> To: thien.m.huynh ;
> thuan.t...@dektech.com.au; minh.c...@dektech.com.au
> Cc: opensaf-devel@lists.sourceforge.net
> Subject: Re: [PATCH 1/1] ntfd: Do not send response to client if
> client down [#3084]
>
> Hi Thien,
>
> I have some comments below.
>
> I see this enhancement does not bring much value to NTF as it deals with a
> very rare case - process is terminated before saNtfInitialize() returns. In
> reality, if NTF server is getting overloaded by such process, there must be
> an error in that process.
>
> @Minh: how about your opinion? is this ticket valid?
>
> Anyway, here are my comments:
> 1) Only C source files, ntfs_mds.c & ntfs_evt.c, access the new added list
> `ntfa_down_list_head`, why put new added methods in the C++ file and add C
> wrapper functions for them?
> It should be more clean if you move these functions into a new files
> e.g: ntfs_client_down.{h,c}.
>
> 2) C++ method name should start with a capital letter (refer to C++
> google coding rule)
>
> 3) Naming methods that represent adding a down client to list, and
> removing from the list should pair/opposite with each other e.g. Open
> vs Close, Add vs Remove, not mark vs remove
>
> 4) The list is accessing from 02 different threads, mds and main thread,
> therefore must use mutex to prevent race conditions.
>
> 5) Should have a check to ensure *not* adding the down client into the list
> if that client has successfully initialized.
>
> Regards, Vu
>
> On 10/9/19 9:36 AM, thien.m.huynh wrote:
>> Ntfd will not send response to a client when client already down.
>> This will avoid timeout when ntfd send via mds.
>> ---
>>src/ntf/ntfd/NtfAdmin.cc | 93
>>
>>src/ntf/ntfd/NtfAdmin.h | 3 ++
>>src/ntf/ntfd/ntfs_cb.h | 6
>>src/ntf/ntfd/ntfs_com.h | 3 ++
>>src/ntf/ntfd/ntfs_evt.c | 1 +
>>src/ntf/ntfd/ntfs_mds.c | 9 -
>>6 files changed, 114 insertions(+), 1 deletion(-)
>>
>> diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc
>> index 8bbee69..641171b 100644
>> --- a/src/ntf/ntfd/NtfAdmin.cc
>> +++ b/src/ntf/ntfd/NtfAdmin.cc
>> @@ -560,6 +560,85 @@ void NtfAdmin::SearchAndSetClientsDownFlag(MDS_DEST
>> mds_dest) {
>>}
>>
>>/**
>> + * @brief Add mds_dest tag into ntfa down list
>> + * @param mds_dest
>> + */
>> +void NtfAdmin::markAgentDown(MDS_DEST mds_dest) {
>> + TRACE_ENTER();
>> + NTFA_DOWN_LIST *ntfa_down_rec = NULL;
>> + if ((ntfa_down_rec = reinterpret_cast(
>> + malloc(sizeof(NTFA_DOWN_LIST == NULL) {
>> +LOG_ER("memory allocation for the NTFA_DOWN_LIST failed");
>> +return;
>> + }
>> + memset(ntfa_down_rec, 0, sizeof(NTFA_DOWN_LIST));
>> + ntfa_down_rec->mds_dest = mds_dest;
>> + ntfa_down_rec->next = NULL;
>> +
>> + if (ntfs_cb->ntfa_down_list_head == NULL) {
>> +ntfs_cb->ntfa_down_list_head = ntfa_down_rec;
>> + } else {
>> +NTFA_DOWN_LIST *p = ntfs_cb->ntfa_down_list_head;
>> +while (p->next != NULL) {
>> + p = p->next;
>> +}
>> +p->next = ntfa_down_rec;
>> + }
>> + TRACE_1("Added MDS dest: %" PRIx64, ntfa_down_rec->mds_dest);
>> + TRACE_LEAVE();
>> +}
>> +
>> +/**
>> + * @brief Find and remove agent from ntfa down list
>> + * @param mds_dest
>> + */
>> +void NtfAdmin::removeAgentFromDownList(MDS_DEST mds_dest) {
>> + NTFA_DOWN_LIST *ntfa_down_rec = ntfs_cb->ntfa_down_list_head;
>> + NTFA_DOWN_LIST *prev = NULL;
>> + TRACE_ENTER();
>> + while (ntfa_down_rec != NULL) {
>> +if (mds_dest == ntfa_down_rec->mds_dest) {
>> + if (ntfa_down_rec == ntfs_cb->ntfa_down_list_head) {
>> +if (ntfa_down_rec->next == NULL) {
>> + ntfs_cb->ntfa_down_list_head = NULL;
>> +} else {
>> + ntfs_cb->ntfa_down_list_head = ntfa_down_rec->next;
>> +}
>> + } else if (prev) {
>> +prev->next =
Re: [devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Hi all,
What I guess from the ticket that it is a race condition between the mds
thread and main thread in ntfd. We normally get NCSDOWN callback from
mds, and send event to main thread to remove the client. But the mds
callback here comes in the middle of processing Initialize().
We have something similar done in ntfd with
SearchAndSetClientsDownFlag(), GetClientDownFlag(), SetClientDownFlag(),
can we try to reuse them?
Thanks,
Minh
On 9/10/19 5:10 pm, Thien Minh Huynh wrote:
Hi Vu,
Thanks for your time to review the patch.
Best Regards,
ThienHuynh
-Original Message-
From: Nguyen Minh Vu
Sent: Wednesday, October 9, 2019 11:15 AM
To: thien.m.huynh ; thuan.t...@dektech.com.au;
minh.c...@dektech.com.au
Cc: opensaf-devel@lists.sourceforge.net
Subject: Re: [PATCH 1/1] ntfd: Do not send response to client if client down
[#3084]
Hi Thien,
I have some comments below.
I see this enhancement does not bring much value to NTF as it deals with a very
rare case - process is terminated before saNtfInitialize() returns. In reality,
if NTF server is getting overloaded by such process, there must be an error in
that process.
@Minh: how about your opinion? is this ticket valid?
Anyway, here are my comments:
1) Only C source files, ntfs_mds.c & ntfs_evt.c, access the new added list
`ntfa_down_list_head`, why put new added methods in the C++ file and add C wrapper
functions for them?
It should be more clean if you move these functions into a new files
e.g: ntfs_client_down.{h,c}.
2) C++ method name should start with a capital letter (refer to C++ google
coding rule)
3) Naming methods that represent adding a down client to list, and removing
from the list should pair/opposite with each other e.g. Open vs Close, Add vs
Remove, not mark vs remove
4) The list is accessing from 02 different threads, mds and main thread,
therefore must use mutex to prevent race conditions.
5) Should have a check to ensure *not* adding the down client into the list if
that client has successfully initialized.
Regards, Vu
On 10/9/19 9:36 AM, thien.m.huynh wrote:
Ntfd will not send response to a client when client already down.
This will avoid timeout when ntfd send via mds.
---
src/ntf/ntfd/NtfAdmin.cc | 93
src/ntf/ntfd/NtfAdmin.h | 3 ++
src/ntf/ntfd/ntfs_cb.h | 6
src/ntf/ntfd/ntfs_com.h | 3 ++
src/ntf/ntfd/ntfs_evt.c | 1 +
src/ntf/ntfd/ntfs_mds.c | 9 -
6 files changed, 114 insertions(+), 1 deletion(-)
diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc index
8bbee69..641171b 100644
--- a/src/ntf/ntfd/NtfAdmin.cc
+++ b/src/ntf/ntfd/NtfAdmin.cc
@@ -560,6 +560,85 @@ void NtfAdmin::SearchAndSetClientsDownFlag(MDS_DEST
mds_dest) {
}
/**
+ * @brief Add mds_dest tag into ntfa down list
+ * @param mds_dest
+ */
+void NtfAdmin::markAgentDown(MDS_DEST mds_dest) {
+ TRACE_ENTER();
+ NTFA_DOWN_LIST *ntfa_down_rec = NULL;
+ if ((ntfa_down_rec = reinterpret_cast(
+ malloc(sizeof(NTFA_DOWN_LIST == NULL) {
+LOG_ER("memory allocation for the NTFA_DOWN_LIST failed");
+return;
+ }
+ memset(ntfa_down_rec, 0, sizeof(NTFA_DOWN_LIST));
+ ntfa_down_rec->mds_dest = mds_dest;
+ ntfa_down_rec->next = NULL;
+
+ if (ntfs_cb->ntfa_down_list_head == NULL) {
+ntfs_cb->ntfa_down_list_head = ntfa_down_rec;
+ } else {
+NTFA_DOWN_LIST *p = ntfs_cb->ntfa_down_list_head;
+while (p->next != NULL) {
+ p = p->next;
+}
+p->next = ntfa_down_rec;
+ }
+ TRACE_1("Added MDS dest: %" PRIx64, ntfa_down_rec->mds_dest);
+ TRACE_LEAVE();
+}
+
+/**
+ * @brief Find and remove agent from ntfa down list
+ * @param mds_dest
+ */
+void NtfAdmin::removeAgentFromDownList(MDS_DEST mds_dest) {
+ NTFA_DOWN_LIST *ntfa_down_rec = ntfs_cb->ntfa_down_list_head;
+ NTFA_DOWN_LIST *prev = NULL;
+ TRACE_ENTER();
+ while (ntfa_down_rec != NULL) {
+if (mds_dest == ntfa_down_rec->mds_dest) {
+ if (ntfa_down_rec == ntfs_cb->ntfa_down_list_head) {
+if (ntfa_down_rec->next == NULL) {
+ ntfs_cb->ntfa_down_list_head = NULL;
+} else {
+ ntfs_cb->ntfa_down_list_head = ntfa_down_rec->next;
+}
+ } else if (prev) {
+prev->next = ntfa_down_rec->next;
+ }
+ TRACE("Deleted MDS dest: %" PRIx64, ntfa_down_rec->mds_dest);
+ free(ntfa_down_rec);
+ ntfa_down_rec = NULL;
+ break;
+}
+prev = ntfa_down_rec;
+ntfa_down_rec = ntfa_down_rec->next;
+ }
+ TRACE_LEAVE();
+}
+
+/**
+ * @brief Check if agent exists in down list
+ * @param mds_dest
+ * @return true/false
+ */
+bool NtfAdmin::isInNtfaDownList(MDS_DEST mds_dest) {
+ bool found = false;
+ NTFA_DOWN_LIST *ntfa_down_rec = ntfs_cb->ntfa_down_list_head;
+ TRACE_ENTER();
+ while (ntfa_down_rec != NULL) {
+if (mds_dest == ntfa_down_rec->mds_dest) {
+ found = true;
+ break;
+}
+ntfa_down_rec = ntfa_down_rec->next;
+ }
+ TRACE_LEAVE();
+
Re: [devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Hi Vu,
Thanks for your time to review the patch.
Best Regards,
ThienHuynh
-Original Message-
From: Nguyen Minh Vu
Sent: Wednesday, October 9, 2019 11:15 AM
To: thien.m.huynh ; thuan.t...@dektech.com.au;
minh.c...@dektech.com.au
Cc: opensaf-devel@lists.sourceforge.net
Subject: Re: [PATCH 1/1] ntfd: Do not send response to client if client down
[#3084]
Hi Thien,
I have some comments below.
I see this enhancement does not bring much value to NTF as it deals with a very
rare case - process is terminated before saNtfInitialize() returns. In reality,
if NTF server is getting overloaded by such process, there must be an error in
that process.
@Minh: how about your opinion? is this ticket valid?
Anyway, here are my comments:
1) Only C source files, ntfs_mds.c & ntfs_evt.c, access the new added list
`ntfa_down_list_head`, why put new added methods in the C++ file and add C
wrapper functions for them?
It should be more clean if you move these functions into a new files
e.g: ntfs_client_down.{h,c}.
2) C++ method name should start with a capital letter (refer to C++ google
coding rule)
3) Naming methods that represent adding a down client to list, and removing
from the list should pair/opposite with each other e.g. Open vs Close, Add vs
Remove, not mark vs remove
4) The list is accessing from 02 different threads, mds and main thread,
therefore must use mutex to prevent race conditions.
5) Should have a check to ensure *not* adding the down client into the list if
that client has successfully initialized.
Regards, Vu
On 10/9/19 9:36 AM, thien.m.huynh wrote:
> Ntfd will not send response to a client when client already down.
> This will avoid timeout when ntfd send via mds.
> ---
> src/ntf/ntfd/NtfAdmin.cc | 93
>
> src/ntf/ntfd/NtfAdmin.h | 3 ++
> src/ntf/ntfd/ntfs_cb.h | 6
> src/ntf/ntfd/ntfs_com.h | 3 ++
> src/ntf/ntfd/ntfs_evt.c | 1 +
> src/ntf/ntfd/ntfs_mds.c | 9 -
> 6 files changed, 114 insertions(+), 1 deletion(-)
>
> diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc index
> 8bbee69..641171b 100644
> --- a/src/ntf/ntfd/NtfAdmin.cc
> +++ b/src/ntf/ntfd/NtfAdmin.cc
> @@ -560,6 +560,85 @@ void NtfAdmin::SearchAndSetClientsDownFlag(MDS_DEST
> mds_dest) {
> }
>
> /**
> + * @brief Add mds_dest tag into ntfa down list
> + * @param mds_dest
> + */
> +void NtfAdmin::markAgentDown(MDS_DEST mds_dest) {
> + TRACE_ENTER();
> + NTFA_DOWN_LIST *ntfa_down_rec = NULL;
> + if ((ntfa_down_rec = reinterpret_cast(
> + malloc(sizeof(NTFA_DOWN_LIST == NULL) {
> +LOG_ER("memory allocation for the NTFA_DOWN_LIST failed");
> +return;
> + }
> + memset(ntfa_down_rec, 0, sizeof(NTFA_DOWN_LIST));
> + ntfa_down_rec->mds_dest = mds_dest;
> + ntfa_down_rec->next = NULL;
> +
> + if (ntfs_cb->ntfa_down_list_head == NULL) {
> +ntfs_cb->ntfa_down_list_head = ntfa_down_rec;
> + } else {
> +NTFA_DOWN_LIST *p = ntfs_cb->ntfa_down_list_head;
> +while (p->next != NULL) {
> + p = p->next;
> +}
> +p->next = ntfa_down_rec;
> + }
> + TRACE_1("Added MDS dest: %" PRIx64, ntfa_down_rec->mds_dest);
> + TRACE_LEAVE();
> +}
> +
> +/**
> + * @brief Find and remove agent from ntfa down list
> + * @param mds_dest
> + */
> +void NtfAdmin::removeAgentFromDownList(MDS_DEST mds_dest) {
> + NTFA_DOWN_LIST *ntfa_down_rec = ntfs_cb->ntfa_down_list_head;
> + NTFA_DOWN_LIST *prev = NULL;
> + TRACE_ENTER();
> + while (ntfa_down_rec != NULL) {
> +if (mds_dest == ntfa_down_rec->mds_dest) {
> + if (ntfa_down_rec == ntfs_cb->ntfa_down_list_head) {
> +if (ntfa_down_rec->next == NULL) {
> + ntfs_cb->ntfa_down_list_head = NULL;
> +} else {
> + ntfs_cb->ntfa_down_list_head = ntfa_down_rec->next;
> +}
> + } else if (prev) {
> +prev->next = ntfa_down_rec->next;
> + }
> + TRACE("Deleted MDS dest: %" PRIx64, ntfa_down_rec->mds_dest);
> + free(ntfa_down_rec);
> + ntfa_down_rec = NULL;
> + break;
> +}
> +prev = ntfa_down_rec;
> +ntfa_down_rec = ntfa_down_rec->next;
> + }
> + TRACE_LEAVE();
> +}
> +
> +/**
> + * @brief Check if agent exists in down list
> + * @param mds_dest
> + * @return true/false
> + */
> +bool NtfAdmin::isInNtfaDownList(MDS_DEST mds_dest) {
> + bool found = false;
> + NTFA_DOWN_LIST *ntfa_down_rec = ntfs_cb->ntfa_down_list_head;
> + TRACE_ENTER();
> + while (ntfa_down_rec != NULL) {
> +if (mds_dest == ntfa_down_rec->mds_dest) {
> + found = true;
> + break;
> +}
> +ntfa_down_rec = ntfa_down_rec->next;
> + }
> + TRACE_LEAVE();
> + return found;
> +}
> +
> +/**
>* The node object where the client who had the subscription is notified
>* so it can delete the appropriate subscription and filter object.
>*
> @@ -1300,6 +1379,20 @@ uint32_t
> send_clm_node_status_change(SaClmClusterChangesT cluster_change,
>
Re: [devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Hi Thien,
I have some comments below.
I see this enhancement does not bring much value to NTF as it deals with
a very rare case -
process is terminated before saNtfInitialize() returns. In reality, if
NTF server is getting overloaded
by such process, there must be an error in that process.
@Minh: how about your opinion? is this ticket valid?
Anyway, here are my comments:
1) Only C source files, ntfs_mds.c & ntfs_evt.c, access the new added
list `ntfa_down_list_head`,
why put new added methods in the C++ file and add C wrapper functions
for them?
It should be more clean if you move these functions into a new files
e.g: ntfs_client_down.{h,c}.
2) C++ method name should start with a capital letter (refer to C++
google coding rule)
3) Naming methods that represent adding a down client to list, and
removing from the list
should pair/opposite with each other e.g. Open vs Close, Add vs Remove,
not mark vs remove
4) The list is accessing from 02 different threads, mds and main thread,
therefore must use mutex
to prevent race conditions.
5) Should have a check to ensure *not* adding the down client into the
list if that client has successfully initialized.
Regards, Vu
On 10/9/19 9:36 AM, thien.m.huynh wrote:
Ntfd will not send response to a client when client already down.
This will avoid timeout when ntfd send via mds.
---
src/ntf/ntfd/NtfAdmin.cc | 93
src/ntf/ntfd/NtfAdmin.h | 3 ++
src/ntf/ntfd/ntfs_cb.h | 6
src/ntf/ntfd/ntfs_com.h | 3 ++
src/ntf/ntfd/ntfs_evt.c | 1 +
src/ntf/ntfd/ntfs_mds.c | 9 -
6 files changed, 114 insertions(+), 1 deletion(-)
diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc
index 8bbee69..641171b 100644
--- a/src/ntf/ntfd/NtfAdmin.cc
+++ b/src/ntf/ntfd/NtfAdmin.cc
@@ -560,6 +560,85 @@ void NtfAdmin::SearchAndSetClientsDownFlag(MDS_DEST
mds_dest) {
}
/**
+ * @brief Add mds_dest tag into ntfa down list
+ * @param mds_dest
+ */
+void NtfAdmin::markAgentDown(MDS_DEST mds_dest) {
+ TRACE_ENTER();
+ NTFA_DOWN_LIST *ntfa_down_rec = NULL;
+ if ((ntfa_down_rec = reinterpret_cast(
+ malloc(sizeof(NTFA_DOWN_LIST == NULL) {
+LOG_ER("memory allocation for the NTFA_DOWN_LIST failed");
+return;
+ }
+ memset(ntfa_down_rec, 0, sizeof(NTFA_DOWN_LIST));
+ ntfa_down_rec->mds_dest = mds_dest;
+ ntfa_down_rec->next = NULL;
+
+ if (ntfs_cb->ntfa_down_list_head == NULL) {
+ntfs_cb->ntfa_down_list_head = ntfa_down_rec;
+ } else {
+NTFA_DOWN_LIST *p = ntfs_cb->ntfa_down_list_head;
+while (p->next != NULL) {
+ p = p->next;
+}
+p->next = ntfa_down_rec;
+ }
+ TRACE_1("Added MDS dest: %" PRIx64, ntfa_down_rec->mds_dest);
+ TRACE_LEAVE();
+}
+
+/**
+ * @brief Find and remove agent from ntfa down list
+ * @param mds_dest
+ */
+void NtfAdmin::removeAgentFromDownList(MDS_DEST mds_dest) {
+ NTFA_DOWN_LIST *ntfa_down_rec = ntfs_cb->ntfa_down_list_head;
+ NTFA_DOWN_LIST *prev = NULL;
+ TRACE_ENTER();
+ while (ntfa_down_rec != NULL) {
+if (mds_dest == ntfa_down_rec->mds_dest) {
+ if (ntfa_down_rec == ntfs_cb->ntfa_down_list_head) {
+if (ntfa_down_rec->next == NULL) {
+ ntfs_cb->ntfa_down_list_head = NULL;
+} else {
+ ntfs_cb->ntfa_down_list_head = ntfa_down_rec->next;
+}
+ } else if (prev) {
+prev->next = ntfa_down_rec->next;
+ }
+ TRACE("Deleted MDS dest: %" PRIx64, ntfa_down_rec->mds_dest);
+ free(ntfa_down_rec);
+ ntfa_down_rec = NULL;
+ break;
+}
+prev = ntfa_down_rec;
+ntfa_down_rec = ntfa_down_rec->next;
+ }
+ TRACE_LEAVE();
+}
+
+/**
+ * @brief Check if agent exists in down list
+ * @param mds_dest
+ * @return true/false
+ */
+bool NtfAdmin::isInNtfaDownList(MDS_DEST mds_dest) {
+ bool found = false;
+ NTFA_DOWN_LIST *ntfa_down_rec = ntfs_cb->ntfa_down_list_head;
+ TRACE_ENTER();
+ while (ntfa_down_rec != NULL) {
+if (mds_dest == ntfa_down_rec->mds_dest) {
+ found = true;
+ break;
+}
+ntfa_down_rec = ntfa_down_rec->next;
+ }
+ TRACE_LEAVE();
+ return found;
+}
+
+/**
* The node object where the client who had the subscription is notified
* so it can delete the appropriate subscription and filter object.
*
@@ -1300,6 +1379,20 @@ uint32_t
send_clm_node_status_change(SaClmClusterChangesT cluster_change,
cluster_change, node_id));
}
+void removeAgentFromDownList(MDS_DEST mds_dest) {
+ osafassert(NtfAdmin::theNtfAdmin != NULL);
+ NtfAdmin::theNtfAdmin->removeAgentFromDownList(mds_dest);
+}
+
+bool isInNtfaDownList(MDS_DEST mds_dest) {
+ return (NtfAdmin::theNtfAdmin->isInNtfaDownList(mds_dest));
+}
+
+void markAgentDown(MDS_DEST mds_dest) {
+ osafassert(NtfAdmin::theNtfAdmin != NULL);
+ NtfAdmin::theNtfAdmin->markAgentDown(mds_dest);
+}
+
/**
* @brief Checks CLM membership status of a client.
* A0101 clients are always CLM member.
diff --git
[devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Ntfd will not send response to a client when client already down.
This will avoid timeout when ntfd send via mds.
---
src/ntf/ntfd/NtfAdmin.cc | 93
src/ntf/ntfd/NtfAdmin.h | 3 ++
src/ntf/ntfd/ntfs_cb.h | 6
src/ntf/ntfd/ntfs_com.h | 3 ++
src/ntf/ntfd/ntfs_evt.c | 1 +
src/ntf/ntfd/ntfs_mds.c | 9 -
6 files changed, 114 insertions(+), 1 deletion(-)
diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc
index 8bbee69..641171b 100644
--- a/src/ntf/ntfd/NtfAdmin.cc
+++ b/src/ntf/ntfd/NtfAdmin.cc
@@ -560,6 +560,85 @@ void NtfAdmin::SearchAndSetClientsDownFlag(MDS_DEST
mds_dest) {
}
/**
+ * @brief Add mds_dest tag into ntfa down list
+ * @param mds_dest
+ */
+void NtfAdmin::markAgentDown(MDS_DEST mds_dest) {
+ TRACE_ENTER();
+ NTFA_DOWN_LIST *ntfa_down_rec = NULL;
+ if ((ntfa_down_rec = reinterpret_cast(
+ malloc(sizeof(NTFA_DOWN_LIST == NULL) {
+LOG_ER("memory allocation for the NTFA_DOWN_LIST failed");
+return;
+ }
+ memset(ntfa_down_rec, 0, sizeof(NTFA_DOWN_LIST));
+ ntfa_down_rec->mds_dest = mds_dest;
+ ntfa_down_rec->next = NULL;
+
+ if (ntfs_cb->ntfa_down_list_head == NULL) {
+ntfs_cb->ntfa_down_list_head = ntfa_down_rec;
+ } else {
+NTFA_DOWN_LIST *p = ntfs_cb->ntfa_down_list_head;
+while (p->next != NULL) {
+ p = p->next;
+}
+p->next = ntfa_down_rec;
+ }
+ TRACE_1("Added MDS dest: %" PRIx64, ntfa_down_rec->mds_dest);
+ TRACE_LEAVE();
+}
+
+/**
+ * @brief Find and remove agent from ntfa down list
+ * @param mds_dest
+ */
+void NtfAdmin::removeAgentFromDownList(MDS_DEST mds_dest) {
+ NTFA_DOWN_LIST *ntfa_down_rec = ntfs_cb->ntfa_down_list_head;
+ NTFA_DOWN_LIST *prev = NULL;
+ TRACE_ENTER();
+ while (ntfa_down_rec != NULL) {
+if (mds_dest == ntfa_down_rec->mds_dest) {
+ if (ntfa_down_rec == ntfs_cb->ntfa_down_list_head) {
+if (ntfa_down_rec->next == NULL) {
+ ntfs_cb->ntfa_down_list_head = NULL;
+} else {
+ ntfs_cb->ntfa_down_list_head = ntfa_down_rec->next;
+}
+ } else if (prev) {
+prev->next = ntfa_down_rec->next;
+ }
+ TRACE("Deleted MDS dest: %" PRIx64, ntfa_down_rec->mds_dest);
+ free(ntfa_down_rec);
+ ntfa_down_rec = NULL;
+ break;
+}
+prev = ntfa_down_rec;
+ntfa_down_rec = ntfa_down_rec->next;
+ }
+ TRACE_LEAVE();
+}
+
+/**
+ * @brief Check if agent exists in down list
+ * @param mds_dest
+ * @return true/false
+ */
+bool NtfAdmin::isInNtfaDownList(MDS_DEST mds_dest) {
+ bool found = false;
+ NTFA_DOWN_LIST *ntfa_down_rec = ntfs_cb->ntfa_down_list_head;
+ TRACE_ENTER();
+ while (ntfa_down_rec != NULL) {
+if (mds_dest == ntfa_down_rec->mds_dest) {
+ found = true;
+ break;
+}
+ntfa_down_rec = ntfa_down_rec->next;
+ }
+ TRACE_LEAVE();
+ return found;
+}
+
+/**
* The node object where the client who had the subscription is notified
* so it can delete the appropriate subscription and filter object.
*
@@ -1300,6 +1379,20 @@ uint32_t
send_clm_node_status_change(SaClmClusterChangesT cluster_change,
cluster_change, node_id));
}
+void removeAgentFromDownList(MDS_DEST mds_dest) {
+ osafassert(NtfAdmin::theNtfAdmin != NULL);
+ NtfAdmin::theNtfAdmin->removeAgentFromDownList(mds_dest);
+}
+
+bool isInNtfaDownList(MDS_DEST mds_dest) {
+ return (NtfAdmin::theNtfAdmin->isInNtfaDownList(mds_dest));
+}
+
+void markAgentDown(MDS_DEST mds_dest) {
+ osafassert(NtfAdmin::theNtfAdmin != NULL);
+ NtfAdmin::theNtfAdmin->markAgentDown(mds_dest);
+}
+
/**
* @brief Checks CLM membership status of a client.
* A0101 clients are always CLM member.
diff --git a/src/ntf/ntfd/NtfAdmin.h b/src/ntf/ntfd/NtfAdmin.h
index 4808ca9..eab0016 100644
--- a/src/ntf/ntfd/NtfAdmin.h
+++ b/src/ntf/ntfd/NtfAdmin.h
@@ -109,6 +109,9 @@ class NtfAdmin {
uint32_t send_cluster_membership_msg_to_clients(
SaClmClusterChangesT cluster_change, NODE_ID node_id);
bool is_stale_client(unsigned int clientId);
+ void markAgentDown(MDS_DEST mds_dest);
+ void removeAgentFromDownList(MDS_DEST mds_dest);
+ bool isInNtfaDownList(MDS_DEST mds_dest);
private:
void processNotification(unsigned int clientId,
diff --git a/src/ntf/ntfd/ntfs_cb.h b/src/ntf/ntfd/ntfs_cb.h
index 96eedc1..518b1b9 100644
--- a/src/ntf/ntfd/ntfs_cb.h
+++ b/src/ntf/ntfd/ntfs_cb.h
@@ -38,6 +38,11 @@ typedef struct {
MDS_DEST mds_dest;
} ntf_client_t;
+typedef struct ntfa_down_list_tag {
+ MDS_DEST mds_dest;
+ struct ntfa_down_list_tag *next;
+} NTFA_DOWN_LIST;
+
typedef struct ntfs_cb {
SYSF_MBX mbx; /* NTFS's mailbox */
MDS_HDL mds_hdl;/* PWE Handle for interacting with NTFAs */
@@ -71,6 +76,7 @@ typedef struct ntfs_cb {
NCS_SEL_OBJ usr2_sel_obj; /* Selection object for CLM initialization.*/
uint16_t peer_mbcsv_version; /*Remeber peer NTFS MBCSV version.*/
bool clm_initialized;
Re: [devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Hi Thuan,
Thanks for your time to review. I will update solution according to your
idea.
Best Regards,
ThienHuynh
-Original Message-
From: Tran Thuan
Sent: Tuesday, October 8, 2019 10:14 AM
To: 'thien.m.huynh' ;
vu.m.ngu...@dektech.com.au; minh.c...@dektech.com.au
Cc: opensaf-devel@lists.sourceforge.net
Subject: RE: [devel] [PATCH 1/1] ntfd: Do not send response to client if
client down [#3084]
Hi Thien,
My yesterday idea seems not right because if agent down before clientAdded()
then it does not work.
Another idea is:
mds_svc_event()
- Handle Agent Down: add call new function markAgentDown() to add
mds_dest to down list
- Handle Agent Up: add call new function markAgentUp() to remove if
mds_dest in down list
ntfs_mds_msg_send()
- Check Agent in down list to skip send message.
Actually similar with your current solution but we need "handle Agent Up"
and check Agent down before sending any msg to agent (more general).
Also, I think markAgentDown() may be new function, not new code in
SearchAndSetClientsDownFlag().
Best Regards,
ThuanTr
-Original Message-
From: Tran Thuan
Sent: Monday, October 7, 2019 5:55 PM
To: 'thien.m.huynh' ;
vu.m.ngu...@dektech.com.au; minh.c...@dektech.com.au
Cc: opensaf-devel@lists.sourceforge.net
Subject: Re: [devel] [PATCH 1/1] ntfd: Do not send response to client if
client down [#3084]
Hi Thien,
I think you just need create a new function
SearchAndGetClientsDownFlag(MDS_DEST mds_dest) Then in
client_added_res_lib(), you will call SearchAndGetClientsDownFlag() to
decide send response or skip.
Best Regards,
ThuanTr
-Original Message-
From: thien.m.huynh
Sent: Monday, October 7, 2019 10:41 AM
To: vu.m.ngu...@dektech.com.au; minh.c...@dektech.com.au
Cc: opensaf-devel@lists.sourceforge.net
Subject: [devel] [PATCH 1/1] ntfd: Do not send response to client if client
down [#3084]
Ntfd will not send response to a client when client already down.
This will avoid timeout when ntfd send via mds.
---
src/ntf/ntfd/NtfAdmin.cc | 61
src/ntf/ntfd/NtfAdmin.h | 1 +
src/ntf/ntfd/ntfs_cb.h | 6 +
src/ntf/ntfd/ntfs_com.c | 4
src/ntf/ntfd/ntfs_com.h | 1 +
src/ntf/ntfd/ntfs_evt.c | 1 +
src/ntf/ntfd/ntfs_mds.c | 2 +-
7 files changed, 75 insertions(+), 1 deletion(-)
diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc index
8bbee69..3a0e9f6 100644
--- a/src/ntf/ntfd/NtfAdmin.cc
+++ b/src/ntf/ntfd/NtfAdmin.cc
@@ -555,6 +555,28 @@ void NtfAdmin::SearchAndSetClientsDownFlag(MDS_DEST
mds_dest) {
client->SetClientDownFlag();
}
}
+
+ CLIENT_DOWN_LIST *client_down_rec = NULL; if ((client_down_rec =
+ reinterpret_cast(
+ malloc(sizeof(CLIENT_DOWN_LIST == NULL) {
+LOG_ER("memory allocation for the CLIENT_DOWN_LIST failed");
+return;
+ }
+ memset(client_down_rec, 0, sizeof(CLIENT_DOWN_LIST));
+ client_down_rec->mds_dest = mds_dest; client_down_rec->next = NULL;
+
+ if (ntfs_cb->client_down_list_head == NULL) {
+ntfs_cb->client_down_list_head = client_down_rec; } else {
+CLIENT_DOWN_LIST *p = ntfs_cb->client_down_list_head;
+while (p->next != NULL) {
+ p = p->next;
+}
+p->next = client_down_rec;
+ }
+ TRACE_1("MDS dest added: %" PRIx64, client_down_rec->mds_dest);
+
osaf_mutex_unlock_ordie(_map_mutex);
TRACE_LEAVE();
}
@@ -1096,6 +1118,41 @@ bool NtfAdmin::is_stale_client(unsigned int
client_id) {
return false;
}
+/**
+ * @brief Checks client added to CLIENT_DOWN_LIST.
+ * Remove client out of list if existed.
+ * @param mds_dest
+ * @return true/false.
+ */
+bool NtfAdmin::checkAddedAndRemove(MDS_DEST mds_dest) {
+ bool found = false;
+ CLIENT_DOWN_LIST *client_down_rec = ntfs_cb->client_down_list_head;
+ CLIENT_DOWN_LIST *prev = NULL;
+ TRACE_ENTER();
+ while (client_down_rec != NULL) {
+if (mds_dest == client_down_rec->mds_dest) {
+ if (client_down_rec == ntfs_cb->client_down_list_head) {
+if (client_down_rec->next == NULL) {
+ ntfs_cb->client_down_list_head = NULL;
+} else {
+ ntfs_cb->client_down_list_head = client_down_rec->next;
+}
+ } else if (prev) {
+prev->next = client_down_rec->next;
+ }
+ TRACE("MDS dest %" PRIx64 " already delete",
client_down_rec->mds_dest);
+ free(client_down_rec);
+ client_down_rec = NULL;
+ found = true;
+ break;
+}
+prev = client_down_rec;
+client_down_rec = client_down_rec->next;
+ }
+ TRACE_LEAVE();
+ return found;
+}
+
// C wrapper funcions start here
/**
@@ -1300,6 +1357,10 @@ uint32_t
send_clm_node_status_change(SaClmClusterChangesT cluster_change,
cluster_change, node_id));
}
+bool checkAddedAndRemove(MDS_DEST mds_dest) {
+ return (NtfAdmin::theNtfAdmin->checkAddedAndRemove(mds_dest));
+}
+
/**
* @brief Checks CLM membership status of a client.
* A0101 clients are always CLM
Re: [devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Hi Thien,
My yesterday idea seems not right because if agent down before clientAdded()
then it does not work.
Another idea is:
mds_svc_event()
- Handle Agent Down: add call new function markAgentDown() to add
mds_dest to down list
- Handle Agent Up: add call new function markAgentUp() to remove if
mds_dest in down list
ntfs_mds_msg_send()
- Check Agent in down list to skip send message.
Actually similar with your current solution but we need "handle Agent Up"
and check Agent down before sending any msg to agent (more general).
Also, I think markAgentDown() may be new function, not new code in
SearchAndSetClientsDownFlag().
Best Regards,
ThuanTr
-Original Message-
From: Tran Thuan
Sent: Monday, October 7, 2019 5:55 PM
To: 'thien.m.huynh' ;
vu.m.ngu...@dektech.com.au; minh.c...@dektech.com.au
Cc: opensaf-devel@lists.sourceforge.net
Subject: Re: [devel] [PATCH 1/1] ntfd: Do not send response to client if
client down [#3084]
Hi Thien,
I think you just need create a new function
SearchAndGetClientsDownFlag(MDS_DEST mds_dest) Then in
client_added_res_lib(), you will call SearchAndGetClientsDownFlag() to
decide send response or skip.
Best Regards,
ThuanTr
-Original Message-
From: thien.m.huynh
Sent: Monday, October 7, 2019 10:41 AM
To: vu.m.ngu...@dektech.com.au; minh.c...@dektech.com.au
Cc: opensaf-devel@lists.sourceforge.net
Subject: [devel] [PATCH 1/1] ntfd: Do not send response to client if client
down [#3084]
Ntfd will not send response to a client when client already down.
This will avoid timeout when ntfd send via mds.
---
src/ntf/ntfd/NtfAdmin.cc | 61
src/ntf/ntfd/NtfAdmin.h | 1 +
src/ntf/ntfd/ntfs_cb.h | 6 +
src/ntf/ntfd/ntfs_com.c | 4
src/ntf/ntfd/ntfs_com.h | 1 +
src/ntf/ntfd/ntfs_evt.c | 1 +
src/ntf/ntfd/ntfs_mds.c | 2 +-
7 files changed, 75 insertions(+), 1 deletion(-)
diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc index
8bbee69..3a0e9f6 100644
--- a/src/ntf/ntfd/NtfAdmin.cc
+++ b/src/ntf/ntfd/NtfAdmin.cc
@@ -555,6 +555,28 @@ void NtfAdmin::SearchAndSetClientsDownFlag(MDS_DEST
mds_dest) {
client->SetClientDownFlag();
}
}
+
+ CLIENT_DOWN_LIST *client_down_rec = NULL; if ((client_down_rec =
+ reinterpret_cast(
+ malloc(sizeof(CLIENT_DOWN_LIST == NULL) {
+LOG_ER("memory allocation for the CLIENT_DOWN_LIST failed");
+return;
+ }
+ memset(client_down_rec, 0, sizeof(CLIENT_DOWN_LIST));
+ client_down_rec->mds_dest = mds_dest; client_down_rec->next = NULL;
+
+ if (ntfs_cb->client_down_list_head == NULL) {
+ntfs_cb->client_down_list_head = client_down_rec; } else {
+CLIENT_DOWN_LIST *p = ntfs_cb->client_down_list_head;
+while (p->next != NULL) {
+ p = p->next;
+}
+p->next = client_down_rec;
+ }
+ TRACE_1("MDS dest added: %" PRIx64, client_down_rec->mds_dest);
+
osaf_mutex_unlock_ordie(_map_mutex);
TRACE_LEAVE();
}
@@ -1096,6 +1118,41 @@ bool NtfAdmin::is_stale_client(unsigned int
client_id) {
return false;
}
+/**
+ * @brief Checks client added to CLIENT_DOWN_LIST.
+ * Remove client out of list if existed.
+ * @param mds_dest
+ * @return true/false.
+ */
+bool NtfAdmin::checkAddedAndRemove(MDS_DEST mds_dest) {
+ bool found = false;
+ CLIENT_DOWN_LIST *client_down_rec = ntfs_cb->client_down_list_head;
+ CLIENT_DOWN_LIST *prev = NULL;
+ TRACE_ENTER();
+ while (client_down_rec != NULL) {
+if (mds_dest == client_down_rec->mds_dest) {
+ if (client_down_rec == ntfs_cb->client_down_list_head) {
+if (client_down_rec->next == NULL) {
+ ntfs_cb->client_down_list_head = NULL;
+} else {
+ ntfs_cb->client_down_list_head = client_down_rec->next;
+}
+ } else if (prev) {
+prev->next = client_down_rec->next;
+ }
+ TRACE("MDS dest %" PRIx64 " already delete",
client_down_rec->mds_dest);
+ free(client_down_rec);
+ client_down_rec = NULL;
+ found = true;
+ break;
+}
+prev = client_down_rec;
+client_down_rec = client_down_rec->next;
+ }
+ TRACE_LEAVE();
+ return found;
+}
+
// C wrapper funcions start here
/**
@@ -1300,6 +1357,10 @@ uint32_t
send_clm_node_status_change(SaClmClusterChangesT cluster_change,
cluster_change, node_id));
}
+bool checkAddedAndRemove(MDS_DEST mds_dest) {
+ return (NtfAdmin::theNtfAdmin->checkAddedAndRemove(mds_dest));
+}
+
/**
* @brief Checks CLM membership status of a client.
* A0101 clients are always CLM member.
diff --git a/src/ntf/ntfd/NtfAdmin.h b/src/ntf/ntfd/NtfAdmin.h index
4808ca9..d0c5528 100644
--- a/src/ntf/ntfd/NtfAdmin.h
+++ b/src/ntf/ntfd/NtfAdmin.h
@@ -109,6 +109,7 @@ class NtfAdmin {
uint32_t send_cluster_membership_msg_to_clients(
SaClmClusterChangesT cluster_change, NODE_ID node_id);
bool is_stale_client(unsigned int clientId);
+ bool checkAddedAndRemove(MDS_DEST mds_dest);
Re: [devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Hi Thien,
I think you just need create a new function
SearchAndGetClientsDownFlag(MDS_DEST mds_dest)
Then in client_added_res_lib(), you will call SearchAndGetClientsDownFlag()
to decide send response or skip.
Best Regards,
ThuanTr
-Original Message-
From: thien.m.huynh
Sent: Monday, October 7, 2019 10:41 AM
To: vu.m.ngu...@dektech.com.au; minh.c...@dektech.com.au
Cc: opensaf-devel@lists.sourceforge.net
Subject: [devel] [PATCH 1/1] ntfd: Do not send response to client if client
down [#3084]
Ntfd will not send response to a client when client already down.
This will avoid timeout when ntfd send via mds.
---
src/ntf/ntfd/NtfAdmin.cc | 61
src/ntf/ntfd/NtfAdmin.h | 1 +
src/ntf/ntfd/ntfs_cb.h | 6 +
src/ntf/ntfd/ntfs_com.c | 4
src/ntf/ntfd/ntfs_com.h | 1 +
src/ntf/ntfd/ntfs_evt.c | 1 +
src/ntf/ntfd/ntfs_mds.c | 2 +-
7 files changed, 75 insertions(+), 1 deletion(-)
diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc index
8bbee69..3a0e9f6 100644
--- a/src/ntf/ntfd/NtfAdmin.cc
+++ b/src/ntf/ntfd/NtfAdmin.cc
@@ -555,6 +555,28 @@ void NtfAdmin::SearchAndSetClientsDownFlag(MDS_DEST
mds_dest) {
client->SetClientDownFlag();
}
}
+
+ CLIENT_DOWN_LIST *client_down_rec = NULL; if ((client_down_rec =
+ reinterpret_cast(
+ malloc(sizeof(CLIENT_DOWN_LIST == NULL) {
+LOG_ER("memory allocation for the CLIENT_DOWN_LIST failed");
+return;
+ }
+ memset(client_down_rec, 0, sizeof(CLIENT_DOWN_LIST));
+ client_down_rec->mds_dest = mds_dest; client_down_rec->next = NULL;
+
+ if (ntfs_cb->client_down_list_head == NULL) {
+ntfs_cb->client_down_list_head = client_down_rec; } else {
+CLIENT_DOWN_LIST *p = ntfs_cb->client_down_list_head;
+while (p->next != NULL) {
+ p = p->next;
+}
+p->next = client_down_rec;
+ }
+ TRACE_1("MDS dest added: %" PRIx64, client_down_rec->mds_dest);
+
osaf_mutex_unlock_ordie(_map_mutex);
TRACE_LEAVE();
}
@@ -1096,6 +1118,41 @@ bool NtfAdmin::is_stale_client(unsigned int
client_id) {
return false;
}
+/**
+ * @brief Checks client added to CLIENT_DOWN_LIST.
+ * Remove client out of list if existed.
+ * @param mds_dest
+ * @return true/false.
+ */
+bool NtfAdmin::checkAddedAndRemove(MDS_DEST mds_dest) {
+ bool found = false;
+ CLIENT_DOWN_LIST *client_down_rec = ntfs_cb->client_down_list_head;
+ CLIENT_DOWN_LIST *prev = NULL;
+ TRACE_ENTER();
+ while (client_down_rec != NULL) {
+if (mds_dest == client_down_rec->mds_dest) {
+ if (client_down_rec == ntfs_cb->client_down_list_head) {
+if (client_down_rec->next == NULL) {
+ ntfs_cb->client_down_list_head = NULL;
+} else {
+ ntfs_cb->client_down_list_head = client_down_rec->next;
+}
+ } else if (prev) {
+prev->next = client_down_rec->next;
+ }
+ TRACE("MDS dest %" PRIx64 " already delete",
client_down_rec->mds_dest);
+ free(client_down_rec);
+ client_down_rec = NULL;
+ found = true;
+ break;
+}
+prev = client_down_rec;
+client_down_rec = client_down_rec->next;
+ }
+ TRACE_LEAVE();
+ return found;
+}
+
// C wrapper funcions start here
/**
@@ -1300,6 +1357,10 @@ uint32_t
send_clm_node_status_change(SaClmClusterChangesT cluster_change,
cluster_change, node_id));
}
+bool checkAddedAndRemove(MDS_DEST mds_dest) {
+ return (NtfAdmin::theNtfAdmin->checkAddedAndRemove(mds_dest));
+}
+
/**
* @brief Checks CLM membership status of a client.
* A0101 clients are always CLM member.
diff --git a/src/ntf/ntfd/NtfAdmin.h b/src/ntf/ntfd/NtfAdmin.h index
4808ca9..d0c5528 100644
--- a/src/ntf/ntfd/NtfAdmin.h
+++ b/src/ntf/ntfd/NtfAdmin.h
@@ -109,6 +109,7 @@ class NtfAdmin {
uint32_t send_cluster_membership_msg_to_clients(
SaClmClusterChangesT cluster_change, NODE_ID node_id);
bool is_stale_client(unsigned int clientId);
+ bool checkAddedAndRemove(MDS_DEST mds_dest);
private:
void processNotification(unsigned int clientId, diff --git
a/src/ntf/ntfd/ntfs_cb.h b/src/ntf/ntfd/ntfs_cb.h index 96eedc1..e09f8fb
100644
--- a/src/ntf/ntfd/ntfs_cb.h
+++ b/src/ntf/ntfd/ntfs_cb.h
@@ -38,6 +38,11 @@ typedef struct {
MDS_DEST mds_dest;
} ntf_client_t;
+typedef struct client_down_list {
+ MDS_DEST mds_dest;
+ struct client_down_list *next;
+} CLIENT_DOWN_LIST;
+
typedef struct ntfs_cb {
SYSF_MBX mbx; /* NTFS's mailbox */
MDS_HDL mds_hdl;/* PWE Handle for interacting with NTFAs */
@@ -71,6 +76,7 @@ typedef struct ntfs_cb {
NCS_SEL_OBJ usr2_sel_obj; /* Selection object for CLM initialization.*/
uint16_t peer_mbcsv_version; /*Remeber peer NTFS MBCSV version.*/
bool clm_initialized;// For CLM init status;
+ CLIENT_DOWN_LIST *client_down_list_head; /* client down reccords */
} ntfs_cb_t;
extern uint32_t ntfs_cb_init(ntfs_cb_t *); diff --git
[devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Ntfd will not send response to a client when client already down.
This will avoid timeout when ntfd send via mds.
---
src/ntf/ntfd/NtfAdmin.cc | 61
src/ntf/ntfd/NtfAdmin.h | 1 +
src/ntf/ntfd/ntfs_cb.h | 6 +
src/ntf/ntfd/ntfs_com.c | 4
src/ntf/ntfd/ntfs_com.h | 1 +
src/ntf/ntfd/ntfs_evt.c | 1 +
src/ntf/ntfd/ntfs_mds.c | 2 +-
7 files changed, 75 insertions(+), 1 deletion(-)
diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc
index 8bbee69..3a0e9f6 100644
--- a/src/ntf/ntfd/NtfAdmin.cc
+++ b/src/ntf/ntfd/NtfAdmin.cc
@@ -555,6 +555,28 @@ void NtfAdmin::SearchAndSetClientsDownFlag(MDS_DEST
mds_dest) {
client->SetClientDownFlag();
}
}
+
+ CLIENT_DOWN_LIST *client_down_rec = NULL;
+ if ((client_down_rec = reinterpret_cast(
+ malloc(sizeof(CLIENT_DOWN_LIST == NULL) {
+LOG_ER("memory allocation for the CLIENT_DOWN_LIST failed");
+return;
+ }
+ memset(client_down_rec, 0, sizeof(CLIENT_DOWN_LIST));
+ client_down_rec->mds_dest = mds_dest;
+ client_down_rec->next = NULL;
+
+ if (ntfs_cb->client_down_list_head == NULL) {
+ntfs_cb->client_down_list_head = client_down_rec;
+ } else {
+CLIENT_DOWN_LIST *p = ntfs_cb->client_down_list_head;
+while (p->next != NULL) {
+ p = p->next;
+}
+p->next = client_down_rec;
+ }
+ TRACE_1("MDS dest added: %" PRIx64, client_down_rec->mds_dest);
+
osaf_mutex_unlock_ordie(_map_mutex);
TRACE_LEAVE();
}
@@ -1096,6 +1118,41 @@ bool NtfAdmin::is_stale_client(unsigned int client_id) {
return false;
}
+/**
+ * @brief Checks client added to CLIENT_DOWN_LIST.
+ * Remove client out of list if existed.
+ * @param mds_dest
+ * @return true/false.
+ */
+bool NtfAdmin::checkAddedAndRemove(MDS_DEST mds_dest) {
+ bool found = false;
+ CLIENT_DOWN_LIST *client_down_rec = ntfs_cb->client_down_list_head;
+ CLIENT_DOWN_LIST *prev = NULL;
+ TRACE_ENTER();
+ while (client_down_rec != NULL) {
+if (mds_dest == client_down_rec->mds_dest) {
+ if (client_down_rec == ntfs_cb->client_down_list_head) {
+if (client_down_rec->next == NULL) {
+ ntfs_cb->client_down_list_head = NULL;
+} else {
+ ntfs_cb->client_down_list_head = client_down_rec->next;
+}
+ } else if (prev) {
+prev->next = client_down_rec->next;
+ }
+ TRACE("MDS dest %" PRIx64 " already delete", client_down_rec->mds_dest);
+ free(client_down_rec);
+ client_down_rec = NULL;
+ found = true;
+ break;
+}
+prev = client_down_rec;
+client_down_rec = client_down_rec->next;
+ }
+ TRACE_LEAVE();
+ return found;
+}
+
// C wrapper funcions start here
/**
@@ -1300,6 +1357,10 @@ uint32_t
send_clm_node_status_change(SaClmClusterChangesT cluster_change,
cluster_change, node_id));
}
+bool checkAddedAndRemove(MDS_DEST mds_dest) {
+ return (NtfAdmin::theNtfAdmin->checkAddedAndRemove(mds_dest));
+}
+
/**
* @brief Checks CLM membership status of a client.
* A0101 clients are always CLM member.
diff --git a/src/ntf/ntfd/NtfAdmin.h b/src/ntf/ntfd/NtfAdmin.h
index 4808ca9..d0c5528 100644
--- a/src/ntf/ntfd/NtfAdmin.h
+++ b/src/ntf/ntfd/NtfAdmin.h
@@ -109,6 +109,7 @@ class NtfAdmin {
uint32_t send_cluster_membership_msg_to_clients(
SaClmClusterChangesT cluster_change, NODE_ID node_id);
bool is_stale_client(unsigned int clientId);
+ bool checkAddedAndRemove(MDS_DEST mds_dest);
private:
void processNotification(unsigned int clientId,
diff --git a/src/ntf/ntfd/ntfs_cb.h b/src/ntf/ntfd/ntfs_cb.h
index 96eedc1..e09f8fb 100644
--- a/src/ntf/ntfd/ntfs_cb.h
+++ b/src/ntf/ntfd/ntfs_cb.h
@@ -38,6 +38,11 @@ typedef struct {
MDS_DEST mds_dest;
} ntf_client_t;
+typedef struct client_down_list {
+ MDS_DEST mds_dest;
+ struct client_down_list *next;
+} CLIENT_DOWN_LIST;
+
typedef struct ntfs_cb {
SYSF_MBX mbx; /* NTFS's mailbox */
MDS_HDL mds_hdl;/* PWE Handle for interacting with NTFAs */
@@ -71,6 +76,7 @@ typedef struct ntfs_cb {
NCS_SEL_OBJ usr2_sel_obj; /* Selection object for CLM initialization.*/
uint16_t peer_mbcsv_version; /*Remeber peer NTFS MBCSV version.*/
bool clm_initialized;// For CLM init status;
+ CLIENT_DOWN_LIST *client_down_list_head; /* client down reccords */
} ntfs_cb_t;
extern uint32_t ntfs_cb_init(ntfs_cb_t *);
diff --git a/src/ntf/ntfd/ntfs_com.c b/src/ntf/ntfd/ntfs_com.c
index 35435bb..ee282dc 100644
--- a/src/ntf/ntfd/ntfs_com.c
+++ b/src/ntf/ntfd/ntfs_com.c
@@ -44,6 +44,10 @@ void client_added_res_lib(SaAisErrorT error, unsigned int
clientId,
ntfsv_msg_t msg;
ntfsv_ckpt_msg_t ckpt;
TRACE_ENTER2("clientId: %u, rv: %u", clientId, error);
+ // Not respond when client already down
+ if (checkAddedAndRemove(mdsDest))
+ return;
+
msg.type = NTFSV_NTFA_API_RESP_MSG;
Re: [devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Hi Thien,
I have a few questions & comments below:
1) Use tab, not spaces to indent C source.
2) What if client down event comes to NTFS while it is handling the
NTFSV_INITIALIZE_REQ msg type?
3) When will a down client be removed from the database
client_down_list_head if the client has successfully done initialization ?
Regards, Vu
On 9/24/19 6:43 PM, thien.m.huynh wrote:
Ntfd will not send response to a client when client already down.
This will avoid timeout when ntfd send via mds.
---
src/ntf/ntfd/NtfAdmin.cc | 26
src/ntf/ntfd/NtfAdmin.h | 1 +
src/ntf/ntfd/ntfs_cb.h | 9
src/ntf/ntfd/ntfs_com.h | 1 +
src/ntf/ntfd/ntfs_evt.c | 53 +++-
5 files changed, 89 insertions(+), 1 deletion(-)
diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc
index 8bbee69..c7754b5 100644
--- a/src/ntf/ntfd/NtfAdmin.cc
+++ b/src/ntf/ntfd/NtfAdmin.cc
@@ -517,6 +517,27 @@ void NtfAdmin::clientRemoveMDS(MDS_DEST mds_dest) {
}
/**
+ * @brief Check clients are exists in clientMap
+ * @param mds_dest
+ * @return true/false.
+ */
+bool NtfAdmin::isClientExisted(MDS_DEST mds_dest) {
+ TRACE_ENTER2("mdsDest: %" PRIx64, mds_dest);
+ bool found = false;
+ auto it = clientMap.begin();
+ while (it != clientMap.end()) {
+NtfClient *client = it->second;
+++it;
+if (client->getMdsDest() == mds_dest) {
+ found = true;
+ break;
+}
+ }
+ TRACE_LEAVE();
+ return found;
+}
+
+/**
* Remove the clients that belong to the ntfa down at standby node.
*
* @param mds_dest
@@ -1177,6 +1198,11 @@ void ClientsDownRemoved(MDS_DEST mds_dest) {
NtfAdmin::theNtfAdmin->ClientsDownRemoved(mds_dest);
}
+bool isClientExisted(MDS_DEST mds_dest) {
+ osafassert(NtfAdmin::theNtfAdmin != NULL);
+ return (NtfAdmin::theNtfAdmin->isClientExisted(mds_dest));
+}
+
void SearchAndSetClientsDownFlag(MDS_DEST mds_dest) {
osafassert(NtfAdmin::theNtfAdmin != NULL);
NtfAdmin::theNtfAdmin->SearchAndSetClientsDownFlag(mds_dest);
diff --git a/src/ntf/ntfd/NtfAdmin.h b/src/ntf/ntfd/NtfAdmin.h
index 4808ca9..ff4b178 100644
--- a/src/ntf/ntfd/NtfAdmin.h
+++ b/src/ntf/ntfd/NtfAdmin.h
@@ -65,6 +65,7 @@ class NtfAdmin {
void notificationLoggedConfirmed(SaNtfIdentifierT notificationId);
void clientRemoved(unsigned int clientId);
void clientRemoveMDS(MDS_DEST mds_dest);
+ bool isClientExisted(MDS_DEST mds_dest);
void ClientsDownRemoved(MDS_DEST mds_dest);
void SearchAndSetClientsDownFlag(MDS_DEST mds_dest);
void subscriptionRemoved(unsigned int clientId,
diff --git a/src/ntf/ntfd/ntfs_cb.h b/src/ntf/ntfd/ntfs_cb.h
index 96eedc1..3b1d715 100644
--- a/src/ntf/ntfd/ntfs_cb.h
+++ b/src/ntf/ntfd/ntfs_cb.h
@@ -38,6 +38,11 @@ typedef struct {
MDS_DEST mds_dest;
} ntf_client_t;
+typedef struct client_down_list {
+ MDS_DEST mds_dest;
+ struct client_down_list *next;
+} CLIENT_DOWN_LIST;
+
typedef struct ntfs_cb {
SYSF_MBX mbx; /* NTFS's mailbox */
MDS_HDL mds_hdl;/* PWE Handle for interacting with NTFAs */
@@ -71,6 +76,10 @@ typedef struct ntfs_cb {
NCS_SEL_OBJ usr2_sel_obj; /* Selection object for CLM initialization.*/
uint16_t peer_mbcsv_version; /*Remeber peer NTFS MBCSV version.*/
bool clm_initialized;// For CLM init status;
+ CLIENT_DOWN_LIST
+ *client_down_list_head; /* client down reccords - fix for not respond to
+ client if client already downs*/
+ bool client_finalized;
} ntfs_cb_t;
extern uint32_t ntfs_cb_init(ntfs_cb_t *);
diff --git a/src/ntf/ntfd/ntfs_com.h b/src/ntf/ntfd/ntfs_com.h
index b9e37da..884eae5 100644
--- a/src/ntf/ntfd/ntfs_com.h
+++ b/src/ntf/ntfd/ntfs_com.h
@@ -77,6 +77,7 @@ void notificationSentConfirmed(unsigned int clientId,
void notificationLoggedConfirmed(SaNtfIdentifierT notificationId);
void clientRemoved(unsigned int clientId);
void clientRemoveMDS(MDS_DEST mds_dest);
+bool isClientExisted(MDS_DEST mds_dest);
void ClientsDownRemoved(MDS_DEST mds_dest);
void SearchAndSetClientsDownFlag(MDS_DEST mds_dest);
void subscriptionRemoved(unsigned int clientId,
diff --git a/src/ntf/ntfd/ntfs_evt.c b/src/ntf/ntfd/ntfs_evt.c
index 19b2f60..7b44cb2 100644
--- a/src/ntf/ntfd/ntfs_evt.c
+++ b/src/ntf/ntfd/ntfs_evt.c
@@ -108,8 +108,34 @@ static uint32_t proc_ntfa_updn_mds_msg(ntfsv_ntfs_evt_t
*evt)
if (ntfs_cb->ha_state == SA_AMF_HA_STANDBY) {
ClientsDownRemoved(evt->fr_dest);
} else {
+if (ntfs_cb->client_finalized == false &&
+!isClientExisted(evt->fr_dest)) {
+CLIENT_DOWN_LIST *client_down_rec = NULL;
+
+if ((client_down_rec = (CLIENT_DOWN_LIST *)malloc(
+ sizeof(CLIENT_DOWN_LIST))) == NULL) {
+LOG_ER("memory allocation for the CLIENT_DOWN_LIST
failed");
+
[devel] [PATCH 1/1] ntfd: Do not send response to client if client down [#3084]
Ntfd will not send response to a client when client already down.
This will avoid timeout when ntfd send via mds.
---
src/ntf/ntfd/NtfAdmin.cc | 26
src/ntf/ntfd/NtfAdmin.h | 1 +
src/ntf/ntfd/ntfs_cb.h | 9
src/ntf/ntfd/ntfs_com.h | 1 +
src/ntf/ntfd/ntfs_evt.c | 53 +++-
5 files changed, 89 insertions(+), 1 deletion(-)
diff --git a/src/ntf/ntfd/NtfAdmin.cc b/src/ntf/ntfd/NtfAdmin.cc
index 8bbee69..c7754b5 100644
--- a/src/ntf/ntfd/NtfAdmin.cc
+++ b/src/ntf/ntfd/NtfAdmin.cc
@@ -517,6 +517,27 @@ void NtfAdmin::clientRemoveMDS(MDS_DEST mds_dest) {
}
/**
+ * @brief Check clients are exists in clientMap
+ * @param mds_dest
+ * @return true/false.
+ */
+bool NtfAdmin::isClientExisted(MDS_DEST mds_dest) {
+ TRACE_ENTER2("mdsDest: %" PRIx64, mds_dest);
+ bool found = false;
+ auto it = clientMap.begin();
+ while (it != clientMap.end()) {
+NtfClient *client = it->second;
+++it;
+if (client->getMdsDest() == mds_dest) {
+ found = true;
+ break;
+}
+ }
+ TRACE_LEAVE();
+ return found;
+}
+
+/**
* Remove the clients that belong to the ntfa down at standby node.
*
* @param mds_dest
@@ -1177,6 +1198,11 @@ void ClientsDownRemoved(MDS_DEST mds_dest) {
NtfAdmin::theNtfAdmin->ClientsDownRemoved(mds_dest);
}
+bool isClientExisted(MDS_DEST mds_dest) {
+ osafassert(NtfAdmin::theNtfAdmin != NULL);
+ return (NtfAdmin::theNtfAdmin->isClientExisted(mds_dest));
+}
+
void SearchAndSetClientsDownFlag(MDS_DEST mds_dest) {
osafassert(NtfAdmin::theNtfAdmin != NULL);
NtfAdmin::theNtfAdmin->SearchAndSetClientsDownFlag(mds_dest);
diff --git a/src/ntf/ntfd/NtfAdmin.h b/src/ntf/ntfd/NtfAdmin.h
index 4808ca9..ff4b178 100644
--- a/src/ntf/ntfd/NtfAdmin.h
+++ b/src/ntf/ntfd/NtfAdmin.h
@@ -65,6 +65,7 @@ class NtfAdmin {
void notificationLoggedConfirmed(SaNtfIdentifierT notificationId);
void clientRemoved(unsigned int clientId);
void clientRemoveMDS(MDS_DEST mds_dest);
+ bool isClientExisted(MDS_DEST mds_dest);
void ClientsDownRemoved(MDS_DEST mds_dest);
void SearchAndSetClientsDownFlag(MDS_DEST mds_dest);
void subscriptionRemoved(unsigned int clientId,
diff --git a/src/ntf/ntfd/ntfs_cb.h b/src/ntf/ntfd/ntfs_cb.h
index 96eedc1..3b1d715 100644
--- a/src/ntf/ntfd/ntfs_cb.h
+++ b/src/ntf/ntfd/ntfs_cb.h
@@ -38,6 +38,11 @@ typedef struct {
MDS_DEST mds_dest;
} ntf_client_t;
+typedef struct client_down_list {
+ MDS_DEST mds_dest;
+ struct client_down_list *next;
+} CLIENT_DOWN_LIST;
+
typedef struct ntfs_cb {
SYSF_MBX mbx; /* NTFS's mailbox */
MDS_HDL mds_hdl;/* PWE Handle for interacting with NTFAs */
@@ -71,6 +76,10 @@ typedef struct ntfs_cb {
NCS_SEL_OBJ usr2_sel_obj; /* Selection object for CLM initialization.*/
uint16_t peer_mbcsv_version; /*Remeber peer NTFS MBCSV version.*/
bool clm_initialized;// For CLM init status;
+ CLIENT_DOWN_LIST
+ *client_down_list_head; /* client down reccords - fix for not respond to
+ client if client already downs*/
+ bool client_finalized;
} ntfs_cb_t;
extern uint32_t ntfs_cb_init(ntfs_cb_t *);
diff --git a/src/ntf/ntfd/ntfs_com.h b/src/ntf/ntfd/ntfs_com.h
index b9e37da..884eae5 100644
--- a/src/ntf/ntfd/ntfs_com.h
+++ b/src/ntf/ntfd/ntfs_com.h
@@ -77,6 +77,7 @@ void notificationSentConfirmed(unsigned int clientId,
void notificationLoggedConfirmed(SaNtfIdentifierT notificationId);
void clientRemoved(unsigned int clientId);
void clientRemoveMDS(MDS_DEST mds_dest);
+bool isClientExisted(MDS_DEST mds_dest);
void ClientsDownRemoved(MDS_DEST mds_dest);
void SearchAndSetClientsDownFlag(MDS_DEST mds_dest);
void subscriptionRemoved(unsigned int clientId,
diff --git a/src/ntf/ntfd/ntfs_evt.c b/src/ntf/ntfd/ntfs_evt.c
index 19b2f60..7b44cb2 100644
--- a/src/ntf/ntfd/ntfs_evt.c
+++ b/src/ntf/ntfd/ntfs_evt.c
@@ -108,8 +108,34 @@ static uint32_t proc_ntfa_updn_mds_msg(ntfsv_ntfs_evt_t
*evt)
if (ntfs_cb->ha_state == SA_AMF_HA_STANDBY) {
ClientsDownRemoved(evt->fr_dest);
} else {
+if (ntfs_cb->client_finalized == false &&
+!isClientExisted(evt->fr_dest)) {
+CLIENT_DOWN_LIST *client_down_rec = NULL;
+
+if ((client_down_rec = (CLIENT_DOWN_LIST *)malloc(
+ sizeof(CLIENT_DOWN_LIST))) == NULL) {
+LOG_ER("memory allocation for the CLIENT_DOWN_LIST
failed");
+break;
+}
+memset(client_down_rec, 0, sizeof(CLIENT_DOWN_LIST));
+
+client_down_rec->mds_dest = evt->fr_dest;
+client_down_rec->next = NULL;
+
+if (ntfs_cb->client_down_list_head == NULL) {
+ntfs_cb->client_down_list_head = client_down_rec;
+} else {
+CLIENT_DOWN_LIST *p =
