Hi Minh,

I have few comments below [Vu] and one question.

I see, in some places,  NTF APIs not always return TRY_AGAIN if both SCs
down.
I am not sure if I feel correctly or not.

E.g: In `saNtfNotificationSend` API
When the client thread comes to code line `ntfa_mds_msg_sync_send()`,
headless occurs. If this is the case, the API may return TIMEOUT.

Regards, Vu.


>-----Original Message-----
>From: Minh Hon Chau [mailto:minh.c...@dektech.com.au]
>Sent: Wednesday, December 23, 2015 11:02 AM
>To: lennart.l...@ericsson.com; praveen.malv...@oracle.com;
>vu.m.ngu...@dektech.com.au
>Cc: opensaf-devel@lists.sourceforge.net
>Subject: [PATCH 1 of 5] NTF: Add support cloud resilience for NTF Agent
[#1180]
>
> osaf/libs/agents/saf/ntfa/ntfa.h      |   31 +-
> osaf/libs/agents/saf/ntfa/ntfa_api.c  |  672
+++++++++++++++++++++++++++----
>--
> osaf/libs/agents/saf/ntfa/ntfa_mds.c  |    6 +-
> osaf/libs/agents/saf/ntfa/ntfa_util.c |  465 ++++++++++++++++++++++-
> 4 files changed, 1022 insertions(+), 152 deletions(-)
>
>
>The patch contains support for cloud resilience feature
>in NTF Agent code. Please refer README.HYDRA for content
>of the changes
>
>diff --git a/osaf/libs/agents/saf/ntfa/ntfa.h
b/osaf/libs/agents/saf/ntfa/ntfa.h
>--- a/osaf/libs/agents/saf/ntfa/ntfa.h
>+++ b/osaf/libs/agents/saf/ntfa/ntfa.h
>@@ -91,6 +91,7 @@ typedef struct ntfa_filter_hdl_rec {
> typedef struct subscriberList {
>       SaNtfHandleT subscriberListNtfHandle;
>       SaNtfSubscriptionIdT subscriberListSubscriptionId;
>+      ntfsv_filter_ptrs_t filters; /* remember the filters used by this
>subscriber */
>       struct subscriberList *prev;
>       struct subscriberList *next;
> } ntfa_subscriber_list_t;
>@@ -100,6 +101,10 @@ typedef struct ntfa_reader_hdl_rec {
>       unsigned int reader_id; /* handle value returned by NTFS for this
client
>*/
>       SaNtfHandleT ntfHandle;
>       unsigned int reader_hdl;        /* READER handle from handle mgr */
>+
>+      ntfsv_filter_ptrs_t filters; /* remember the filters used by this
reader
>*/
>+      SaNtfSearchCriteriaT searchCriteria; /* remember the searchCriteria
for
>recovery */
>+
>       struct ntfa_reader_hdl_rec *next;       /* next pointer for the list
in
>ntfa_cb_t */
>       struct ntfa_client_hdl_rec *parent_hdl; /* Back Pointer to the
client
>instantiation */
> } ntfa_reader_hdl_rec_t;
>@@ -114,24 +119,35 @@ typedef struct ntfa_client_hdl_rec {
>       ntfa_reader_hdl_rec_t *reader_list;
>       SYSF_MBX mbx;           /* priority q mbx b/w MDS & Library */
>       struct ntfa_client_hdl_rec *next;       /* next pointer for the list
in
>ntfa_cb_t */
>+      bool valid;             /* handle is valid if it's known by NTF
server,
>used for headless hydra */
>+      SaVersionT version; /* the API version is being used by client, used
for
>recover after headless */
> } ntfa_client_hdl_rec_t;
>
> /*
>  * The NTFA control block is the master anchor structure for all NTFA
>  * instantiations within a process.
>  */
>+typedef enum {
>+      NTFA_NTFSV_NONE = 0,
>+      NTFA_NTFSV_DOWN,
>+      NTFA_NTFSV_NO_ACTIVE,
>+      NTFA_NTFSV_NEW_ACTIVE,
>+      NTFA_NTFSV_UP
>+}ntfa_ntfsv_state_t;
>+
> typedef struct {
>       pthread_mutex_t cb_lock;        /* CB lock */
>       ntfa_client_hdl_rec_t *client_list;     /* NTFA client handle
database
>*/
>       ntfa_reader_hdl_rec_t *reader_list;
>       MDS_HDL mds_hdl;        /* MDS handle */
>       MDS_DEST ntfs_mds_dest; /* NTFS absolute/virtual address */
>-      int ntfs_up;            /* Indicate that MDS subscription
>-                               * is complete */
>+
>       /* NTFS NTFA sync params */
>       int ntfs_sync_awaited;
>       NCS_SEL_OBJ ntfs_sync_sel;
>       SaUint32T ntf_var_data_limit;   /* max allowed variableDataSize */
>+      /* NTF Server state */
>+      ntfa_ntfsv_state_t ntfa_ntfsv_state;
> } ntfa_cb_t;
>
> /* ntfa_saf_api.c */
>@@ -149,7 +165,7 @@ extern void ntfsv_ntfa_evt_free(struct n
>
> /* ntfa_init.c */
> extern unsigned int ntfa_startup(void);
>-extern unsigned int ntfa_shutdown(void);
>+extern unsigned int ntfa_shutdown(bool forced);
>
> /* ntfa_hdl.c */
> extern SaAisErrorT ntfa_hdl_cbk_dispatch(ntfa_cb_t *,
ntfa_client_hdl_rec_t *,
>SaDispatchFlagsT);
>@@ -159,6 +175,7 @@ extern ntfa_notification_hdl_rec_t *ntfa
> extern ntfa_filter_hdl_rec_t
*ntfa_filter_hdl_rec_add(ntfa_client_hdl_rec_t
>**hdl_rec);
> extern void ntfa_hdl_list_del(ntfa_client_hdl_rec_t **);
> extern uint32_t ntfa_hdl_rec_del(ntfa_client_hdl_rec_t **,
>ntfa_client_hdl_rec_t *);
>+extern void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **,
>ntfa_client_hdl_rec_t *);
> extern uint32_t ntfa_notification_hdl_rec_del(ntfa_notification_hdl_rec_t
**,
>ntfa_notification_hdl_rec_t *);
> extern uint32_t ntfa_filter_hdl_rec_del(ntfa_filter_hdl_rec_t **,
>ntfa_filter_hdl_rec_t *);
> extern bool ntfa_validate_ntfa_client_hdl(ntfa_cb_t *ntfa_cb,
>ntfa_client_hdl_rec_t *find_hdl_rec);
>@@ -166,11 +183,15 @@ extern bool ntfa_validate_ntfa_client_hd
> /* ntfa_util.c */
> extern ntfa_client_hdl_rec_t *ntfa_find_hdl_rec_by_client_id(ntfa_cb_t
>*ntfa_cb, uint32_t client_id);
> extern void ntfa_msg_destroy(ntfsv_msg_t *msg);
>-extern void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t
*instance);
>-extern void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t
>+extern void ntfa_notification_destructor(ntfa_notification_hdl_rec_t
>*instance);
>+extern void ntfa_filter_destructor(ntfa_filter_hdl_rec_t
>                                          *notificationFilterInstance);
> extern ntfa_reader_hdl_rec_t
*ntfa_reader_hdl_rec_add(ntfa_client_hdl_rec_t
>**hdl_rec);
> extern uint32_t ntfa_reader_hdl_rec_del(ntfa_reader_hdl_rec_t **,
>ntfa_reader_hdl_rec_t *);
> extern void ntfa_add_to_async_cbk_msg_list(ntfsv_msg_t ** head,
>ntfsv_msg_t * new_node);
> extern uint32_t ntfa_ntfs_msg_proc(ntfa_cb_t *cb, ntfsv_msg_t *ntfsv_msg,
>MDS_SEND_PRIORITY_TYPE prio);
>+extern void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState);
>+extern SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
>+                                                              const
>ntfsv_filter_ptrs_t* pSrc);
>+extern SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t*
filter_ptrs);
> #endif   /* !NTFA_H */
>diff --git a/osaf/libs/agents/saf/ntfa/ntfa_api.c
>b/osaf/libs/agents/saf/ntfa/ntfa_api.c
>--- a/osaf/libs/agents/saf/ntfa/ntfa_api.c
>+++ b/osaf/libs/agents/saf/ntfa/ntfa_api.c
>@@ -34,11 +34,42 @@
> /* The main controle block */
> ntfa_cb_t ntfa_cb = {
>       .cb_lock = PTHREAD_MUTEX_INITIALIZER,
>+      .ntfa_ntfsv_state = NTFA_NTFSV_NONE,
> };
>
> /* list of subscriptions for this process */
> ntfa_subscriber_list_t *subscriberNoList = NULL;
>
>+/*
>+ * @Brief: Determine the common "availability" of API, which depends on
the
>+ *         NTF Server state
>+ * @Param: None
>+ * @Return: OK - If Server state is UP
>+ *          TRY_AGAIN - If Server state is No Active (temporary no active)
>+ *          BAD_HANDLE - If Server state is completely Down (no active at
all).
>+ *                       This error code will be returned to client
calling API
>+ */
>+static SaAisErrorT checkNtfServerState()
>+{
>+      SaAisErrorT rc = SA_AIS_ERR_NOT_SUPPORTED;
>+      /* Check NTF server availability */
>+      if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_UP) {
>+              rc = SA_AIS_OK;
>+      } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_DOWN) {
>+              TRACE("NTFS server is down");
>+              rc = SA_AIS_ERR_TRY_AGAIN;
>+      } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NO_ACTIVE) {
>+              TRACE("NTFS server is unavailable");
>+              rc = SA_AIS_ERR_TRY_AGAIN;
>+      } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NONE) {
>+              TRACE("No NTF server is detected, or API is called during
>headless");
>+              rc = SA_AIS_ERR_TRY_AGAIN;
>+      } else
>+              TRACE("Not supported API call under NTF Server state (%u)",
>+                              ntfa_cb.ntfa_ntfsv_state);
>+      return rc;
>+}
>+
> static SaAisErrorT checkNtfValueTypeRange(SaNtfValueTypeT type)
> {
>       return (type < SA_NTF_VALUE_UINT8 || type >
>SA_NTF_VALUE_ARRAY)? SA_AIS_ERR_INVALID_PARAM
>@@ -871,7 +902,266 @@ SaAisErrorT getFilters(const SaNtfNotifi
> }
>
> /* end help functions */
>-
>+/****************************************************************
>************
>+  Name          : reinitializeClient
>+
>+  Description   : This routine sends initialize messages to NTF Server in
>+                order to obtain new client id
>+
>+  Arguments     : client_hdl* [IN]: client handle
>+
>+  Return Values : SA_AIS_OK if success, others are failure
>+
>+  Notes         : None
>+*****************************************************************
>*************/
>+SaAisErrorT reinitializeClient(ntfa_client_hdl_rec_t* client_hdl) {
[Vu] This function and other ones, recoverReader/Client/Subscriber are only
used internally in ntfa_api.c file.
Should define them as `static` functions?

>+      uint32_t mds_rc;
>+      SaAisErrorT rc = SA_AIS_OK;
>+      ntfsv_msg_t i_msg, *o_msg = NULL;
>+
>+      TRACE_ENTER();
>+
>+      memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>+      i_msg.type = NTFSV_NTFA_API_MSG;
>+      i_msg.info.api_info.type = NTFSV_INITIALIZE_REQ;
>+      i_msg.info.api_info.param.init.version = client_hdl->version;
>+
>+      mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>NTFS_WAIT_TIME);
>+      switch (mds_rc) {
>+      case NCSCC_RC_SUCCESS:
>+              break;
>+      case NCSCC_RC_REQ_TIMOUT:
>+              rc = SA_AIS_ERR_TRY_AGAIN;
>+              goto done;
>+      default:
>+              rc = SA_AIS_ERR_BAD_HANDLE;
>+              goto done;
>+      }
>+
>+      osafassert(o_msg != NULL);
>+      /* Check response msg: type, rc */
>+      if (o_msg->info.api_resp_info.type != NTFSV_INITIALIZE_RSP) {
>+              TRACE("info.api_resp_info.type:%u", o_msg-
>>info.api_resp_info.type);
>+              rc = SA_AIS_ERR_LIBRARY;
>+              goto done;
>+      }
>+      if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>+              TRACE("info.api_resp_info.rc:%u", o_msg-
>>info.api_resp_info.rc);
>+              rc = SA_AIS_ERR_BAD_HANDLE;
>+              goto done;
>+      }
[Vu] Is there any reason not return error code `rc`, but BAD_HANDLE here?

>+
>+      client_hdl->ntfs_client_id = o_msg-
>>info.api_resp_info.param.init_rsp.client_id;
>+      TRACE("Successfully recover client_id, new client_id:%u",
>+                      client_hdl->ntfs_client_id);
>+
>+done:
>+      if (o_msg)
>+              ntfa_msg_destroy(o_msg);
>+      TRACE_LEAVE();
>+      return rc;
>+}
>+/****************************************************************
>************
>+  Name          : recoverReader
>+
>+  Description   : This routine sends read_initialize messages to NTF
Server in
>+                order to reintroduce this reader to NTF Server
>+
>+  Arguments     : client_hdl* [IN]: client handle
>+                reader_hdl* [IN]: reader handle
>+
>+  Return Values : SA_AIS_OK if success, others are failure
>+
>+  Notes         : None
>+*****************************************************************
>*************/
>+SaAisErrorT recoverReader(ntfa_client_hdl_rec_t* client_hdl,
>ntfa_reader_hdl_rec_t* reader_hdl) {
>+      uint32_t mds_rc;
>+      SaAisErrorT rc = SA_AIS_OK;
>+      ntfsv_msg_t i_msg, *o_msg = NULL;
>+      ntfsv_reader_init_req_2_t *send_param;
>+
>+      TRACE_ENTER();
>+
>+      /* Make sure the filters have not been deleted */
>+      if (reader_hdl->filters.alarm_filter == NULL &&
>+              reader_hdl->filters.att_ch_filter == NULL &&
>+              reader_hdl->filters.obj_cr_del_filter == NULL &&
>+              reader_hdl->filters.sec_al_filter == NULL &&
>+              reader_hdl->filters.sta_ch_filter == NULL)
>+              return SA_AIS_ERR_BAD_HANDLE;
>+
>+      memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>+      i_msg.type = NTFSV_NTFA_API_MSG;
>+      i_msg.info.api_info.type = NTFSV_READER_INITIALIZE_REQ_2;
>+      send_param = &i_msg.info.api_info.param.reader_init_2;
>+      send_param->head.client_id = client_hdl->ntfs_client_id;
>+      send_param->head.searchCriteria = reader_hdl->searchCriteria;
>+      send_param->f_rec = reader_hdl->filters;
>+
>+      mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>NTFS_WAIT_TIME);
>+
>+      switch (mds_rc) {
>+      case NCSCC_RC_SUCCESS:
>+              break;
>+      case NCSCC_RC_REQ_TIMOUT:
>+              rc = SA_AIS_ERR_TRY_AGAIN;
>+              goto done;
>+      default:
>+              rc = SA_AIS_ERR_BAD_HANDLE;
>+              goto done;
>+      }
>+
>+      osafassert(o_msg != NULL);
>+      if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>+              TRACE("o_msg->info.api_resp_info.rc:%u", o_msg-
>>info.api_resp_info.rc);
>+              rc = SA_AIS_ERR_BAD_HANDLE;
>+              goto done;
>+      }
>+
>+      if (o_msg->info.api_resp_info.type != NTFSV_READER_INITIALIZE_RSP) {
>+              TRACE("msg type (%d) failed", (int)o_msg-
>>info.api_resp_info.type);
>+              rc = SA_AIS_ERR_LIBRARY;
>+              goto done;
>+      }
>+      /* Update reader_id since it may be changed */
>+      reader_hdl->reader_id = o_msg-
>>info.api_resp_info.param.reader_init_rsp.readerId;
>+
>+      TRACE("Recover reader successfully");
>+done:
>+      if (o_msg)
>+              ntfa_msg_destroy(o_msg);
>+      TRACE_LEAVE();
>+      return rc;
>+}
>+/****************************************************************
>************
>+  Name          : recoverSubscriber
>+
>+  Description   : This routine sends subscribe messages to NTF Server in
>+                order to reintroduce this subscriber to NTF Server
>+
>+  Arguments     : client_hdl* [IN]: client handle
>+                subscriber_hdl* [IN]: subscriber handle
>+
>+  Return Values : SA_AIS_OK if success, others are failure
>+
>+  Notes         : None
>+*****************************************************************
>*************/
>+SaAisErrorT recoverSubscriber(ntfa_client_hdl_rec_t* client_hdl,
>+
>       ntfa_subscriber_list_t* subscriber_hdl) {
>+      uint32_t mds_rc;
>+      SaAisErrorT rc = SA_AIS_OK;
>+      ntfsv_msg_t i_msg, *o_msg = NULL;
>+      ntfsv_subscribe_req_t *send_param;
>+
>+      TRACE_ENTER();
>+
>+      /* Make sure the filters have not been deleted */
>+      if (subscriber_hdl->filters.alarm_filter == NULL &&
>+              subscriber_hdl->filters.att_ch_filter == NULL &&
>+              subscriber_hdl->filters.obj_cr_del_filter == NULL &&
>+              subscriber_hdl->filters.sec_al_filter == NULL &&
>+              subscriber_hdl->filters.sta_ch_filter == NULL)
>+              return SA_AIS_ERR_BAD_HANDLE;
>+
>+      memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>+      i_msg.type = NTFSV_NTFA_API_MSG;
>+      i_msg.info.api_info.type = NTFSV_SUBSCRIBE_REQ;
>+      send_param = &i_msg.info.api_info.param.subscribe;
>+
>+      send_param->client_id = client_hdl->ntfs_client_id;
>+      send_param->subscriptionId = subscriber_hdl-
>>subscriberListSubscriptionId;
>+      send_param->f_rec = subscriber_hdl->filters;
>+
>+      mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>NTFS_WAIT_TIME);
>+
>+      switch (mds_rc) {
>+      case NCSCC_RC_SUCCESS:
>+              break;
>+      case NCSCC_RC_REQ_TIMOUT:
>+              rc = SA_AIS_ERR_TRY_AGAIN;
>+              goto done;
>+      default:
>+              rc = SA_AIS_ERR_BAD_HANDLE;
>+              goto done;
>+      }
>+
>+      osafassert(o_msg != NULL);
>+
>+      if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>+              TRACE("o_msg->info.api_resp_info.rc:%u", o_msg-
>>info.api_resp_info.rc);
>+              rc = SA_AIS_ERR_BAD_HANDLE;
>+              goto done;
>+      }
>+
>+      if (o_msg->info.api_resp_info.type != NTFSV_SUBSCRIBE_RSP) {
>+              TRACE("msg type (%d) failed", (int)o_msg-
>>info.api_resp_info.type);
>+              rc = SA_AIS_ERR_LIBRARY;
>+              goto done;
>+      }
>+      TRACE("Recover subscriber successfully");
>+done:
>+      if (o_msg)
>+              ntfa_msg_destroy(o_msg);
>+
>+      TRACE_LEAVE();
>+      return rc;
>+}
>+/****************************************************************
>************
>+  Name          : recoverClient
>+
>+  Description   : In NTF-A.03.01, section 2.1.2. A client can be producer,
>+                subscriber, and reader at the same time.
>+                This routine will recovery:
>+                (1) client_id
>+                (2) Then it will continue the recovery if this client also
>+                has instance of subscriber or reader.
>+  Arguments     : client_hdl* [IN]: client handle
>+
>+  Return Values : SA_AIS_OK if success, others are failure
>+
>+  Notes         : None
>+*****************************************************************
>*************/
>+SaAisErrorT recoverClient(ntfa_client_hdl_rec_t *client_hdl) {
>+
>+      TRACE_ENTER();
>+      SaAisErrorT rc = SA_AIS_OK;
>+
>+      osafassert(client_hdl);
>+
>+      if ((rc = reinitializeClient(client_hdl)) == SA_AIS_OK) {
>+              /* Restore reader */
>+              ntfa_reader_hdl_rec_t* reader_hdl = client_hdl->reader_list;
>+              while (reader_hdl != NULL && rc == SA_AIS_OK) {
>+                      rc = recoverReader(client_hdl, reader_hdl);
>+                      reader_hdl = reader_hdl->next;
>+              }
>+              if (rc != SA_AIS_OK) {
>+                      TRACE("Failed to restore reader (readerId:%d)",
>+                                      reader_hdl->reader_id);
>+                      goto done;
>+              }
>+              /* Restore subscriber */
>+              ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
>+              while (subscriber_hdl != NULL && rc == SA_AIS_OK) {
>+                      if (client_hdl->local_hdl == subscriber_hdl-
>>subscriberListNtfHandle)
>+                              rc = recoverSubscriber(client_hdl,
>subscriber_hdl);
>+                      subscriber_hdl = subscriber_hdl->next;
>+              }
>+              if (rc != SA_AIS_OK) {
>+                      TRACE("Failed to restore subscriber
>(subscriptionId:%d)",
>+                                      subscriber_hdl-
>>subscriberListSubscriptionId);
>+                      goto done;
>+              }
>+              client_hdl->valid = true;
>+      } else {
>+              TRACE("Failed to restore client (id:%d)", client_hdl-
>>ntfs_client_id);
>+              goto done;
>+      }
>+
>+done:
>+      TRACE_LEAVE();
>+      return rc;
>+}
>
>/*****************************************************************
>**********
>  * 8.4.1
>  *
>@@ -923,15 +1213,14 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>               version->releaseCode = NTF_RELEASE_CODE;
>               version->majorVersion = NTF_MAJOR_VERSION;
>               version->minorVersion = NTF_MINOR_VERSION;
>-              ntfa_shutdown();
>+              ntfa_shutdown(false);
>               rc = SA_AIS_ERR_VERSION;
>               goto done;
>       }
>
>-      if (!ntfa_cb.ntfs_up) {
>-              ntfa_shutdown();
>-              TRACE("NTFS server is down");
>-              rc = SA_AIS_ERR_TRY_AGAIN;
>+      /* Check NTF server availability */
>+      if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>+              ntfa_shutdown(false);
>               goto done;
>       }
>
>@@ -977,7 +1266,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>               rc = SA_AIS_ERR_NO_MEMORY;
>               goto err;
>       }
>-
>+      ntfa_hdl_rec->version = *version;
>       /* pass the handle value to the appl */
>       if (SA_AIS_OK == rc)
>               *ntfHandle = ntfa_hdl_rec->local_hdl;
>@@ -990,7 +1279,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>
>       if (rc != SA_AIS_OK) {
>               TRACE_2("NTFA INIT FAILED\n");
>-              ntfa_shutdown();
>+              ntfa_shutdown(false);
>       }
>
>  done:
>@@ -1101,6 +1390,28 @@ SaAisErrorT saNtfDispatch(SaNtfHandleT n
>               goto done;
>       }
>
>+      /* Need to check NTF server availability here
>+       * Because if the notificationCallback just comes after MDS_DOWN
>+       * it will go to recovery then result in failure due to no director
>+       */
>+      if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>+              ncshm_give_hdl(ntfHandle);
>+              goto done;
>+      }
>+
>+      if (!hdl_rec->valid) {
>+              /* recovery */
>+              if ((rc = recoverClient(hdl_rec)) != SA_AIS_OK) {
>+                      if (rc == SA_AIS_ERR_BAD_HANDLE) {
>+                              ncshm_give_hdl(ntfHandle);
>+
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>+                              ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
>hdl_rec);
>+
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>+                              ntfa_shutdown(false);
>+                              goto done;
>+                      }
>+              }
>+      }
>       if ((rc = ntfa_hdl_cbk_dispatch(&ntfa_cb, hdl_rec, dispatchFlags))
!=
>SA_AIS_OK)
>               TRACE("NTFA_DISPATCH_FAILURE");
>
>@@ -1145,6 +1456,13 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>
>       TRACE_ENTER();
>
>+      /* Check NTF server availability */
>+      if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NO_ACTIVE) {
>+              TRACE("NTFS server is temporarily unavailable");
>+              rc = SA_AIS_ERR_TRY_AGAIN;
[Vu] Better to unify using `checkNtfServerState()`?
>+              goto done;
>+      }
>+
>       /* retrieve hdl rec */
>       hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA, ntfHandle);
>       if (hdl_rec == NULL) {
>@@ -1152,43 +1470,37 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>               rc = SA_AIS_ERR_BAD_HANDLE;
>               goto done;
>       }
>-
>-      /* Check Whether NTFS is up or not */
>-      if (!ntfa_cb.ntfs_up) {
>-              TRACE("NTFS down");
>-              rc = SA_AIS_ERR_TRY_AGAIN;
>-              goto done_give_hdl;
>+      if (hdl_rec->valid) {
>+              /** populate & send the finalize message
>+               ** and make sure the finalize from the server
>+               ** end returned before deleting the local records.
>+               **/
>+              memset(&msg, 0, sizeof(ntfsv_msg_t));
>+              msg.type = NTFSV_NTFA_API_MSG;
>+              msg.info.api_info.type = NTFSV_FINALIZE_REQ;
>+              msg.info.api_info.param.finalize.client_id = hdl_rec-
>>ntfs_client_id;
>+
>+              mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>NTFS_WAIT_TIME);
>+              switch (mds_rc) {
>+              case NCSCC_RC_SUCCESS:
>+                      break;
>+              case NCSCC_RC_REQ_TIMOUT:
>+                      rc = SA_AIS_ERR_TIMEOUT;
>+                      TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>+                      goto done_give_hdl;
>+              default:
>+                      TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>+                      rc = SA_AIS_ERR_NO_RESOURCES;
>+                      goto done_give_hdl;
>+              }
>+
>+              if (o_msg != NULL) {
>+                      rc = o_msg->info.api_resp_info.rc;
>+                      ntfa_msg_destroy(o_msg);
>+              } else
>+                      rc = SA_AIS_ERR_NO_RESOURCES;
>       }
>
>-    /** populate & send the finalize message
>-     ** and make sure the finalize from the server
>-     ** end returned before deleting the local records.
>-     **/
>-      memset(&msg, 0, sizeof(ntfsv_msg_t));
>-      msg.type = NTFSV_NTFA_API_MSG;
>-      msg.info.api_info.type = NTFSV_FINALIZE_REQ;
>-      msg.info.api_info.param.finalize.client_id =
hdl_rec->ntfs_client_id;
>-
>-      mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>NTFS_WAIT_TIME);
>-      switch (mds_rc) {
>-      case NCSCC_RC_SUCCESS:
>-              break;
>-      case NCSCC_RC_REQ_TIMOUT:
>-              rc = SA_AIS_ERR_TIMEOUT;
>-              TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>-              goto done_give_hdl;
>-      default:
>-              TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>-              rc = SA_AIS_ERR_NO_RESOURCES;
>-              goto done_give_hdl;
>-      }
>-
>-      if (o_msg != NULL) {
>-              rc = o_msg->info.api_resp_info.rc;
>-              ntfa_msg_destroy(o_msg);
>-      } else
>-              rc = SA_AIS_ERR_NO_RESOURCES;
>-
>       if (rc == SA_AIS_OK) {
>       /** delete the hdl rec
>          ** including all resources allocated by client if MDS send is
>@@ -1205,7 +1517,7 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>       ncshm_give_hdl(ntfHandle);
>
>       if (rc == SA_AIS_OK) {
>-              rc = ntfa_shutdown();
>+              rc = ntfa_shutdown(false);
>               if (rc != NCSCC_RC_SUCCESS)
>                       TRACE_1("ntfa_shutdown failed");
>       }
>@@ -1349,7 +1661,7 @@ SaAisErrorT saNtfNotificationFree(SaNtfN
>       }
>
>       /* free the resources allocated by
saNtf<ntfType>NotificationAllocate
>*/
>-      ntfa_hdl_rec_destructor(notification_hdl_rec);
>+      ntfa_notification_destructor(notification_hdl_rec);
>
>     /** Delete the resources related to the notificationHandle &
>      *  remove reference in the client.
>@@ -1419,6 +1731,24 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>       }
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>
>+      /* Check NTF server availability */
>+      if ((rc = checkNtfServerState()) != SA_AIS_OK)
>+              goto done_give_hdls;
>+
>+      /* Recover if this is invalid handle */
>+      if (!client_rec->valid) {
>+              if ((rc = recoverClient(client_rec)) != SA_AIS_OK) {
>+                      ncshm_give_hdl(client_handle);
>+                      ncshm_give_hdl(notificationHandle);
>+                      if (rc == SA_AIS_ERR_BAD_HANDLE) {
>+
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>+                              ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
>client_rec);
>+
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>+                              ntfa_shutdown(false);
>+                      }
>+                      goto err_free;
>+              }
>+      }
>     /**
>      ** Populate a sync MDS message
>      **/
>@@ -1481,13 +1811,6 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>               goto done_give_hdls;
>       }
>
>-      /* Check whether NTFS is up or not */
>-      if (!ntfa_cb.ntfs_up) {
>-              TRACE("NTFS down");
>-              rc = SA_AIS_ERR_TRY_AGAIN;
>-              goto done_give_hdls;
>-      }
>-
>       /* Send a sync MDS message to obtain a notification id */
>       mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>timeout);
>       switch (mds_rc) {
>@@ -1596,7 +1919,8 @@ static  SaNtfHandleT ntfHandleGet(SaNtfS
>       return ntfHandle;
> }
>
>-static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
>SaNtfSubscriptionIdT subscriptionId)
>+static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
>SaNtfSubscriptionIdT subscriptionId,
>+
>       ntfsv_filter_ptrs_t filters)
> {
>       SaAisErrorT rc = SA_AIS_OK;
>       ntfa_subscriber_list_t* ntfSubscriberList;
>@@ -1610,7 +1934,9 @@ static SaAisErrorT subscriptionListAdd(S
>       /* Add ntfHandle and subscriptionId into list */
>       ntfSubscriberList->subscriberListNtfHandle = ntfHandle;
>       ntfSubscriberList->subscriberListSubscriptionId = subscriptionId;
>-
>+      memset(&ntfSubscriberList->filters, 0, sizeof(ntfsv_filter_ptrs_t));
>+      ntfa_copy_ntf_filter_ptrs(&ntfSubscriberList->filters, &filters);
>+
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>       if (NULL == subscriberNoList) {
>               subscriberNoList = ntfSubscriberList;
>@@ -1648,6 +1974,7 @@ static void subscriberListItemRemove(SaN
>                       subscriberNoList = NULL;
>       }
>       TRACE_1("REMOVE: listPtr->SubscriptionId %d", listPtr-
>>subscriberListSubscriptionId);
>+      ntfa_del_ntf_filter_ptrs(&listPtr->filters);
>       free(listPtr);
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> }
>@@ -1666,20 +1993,33 @@ SaAisErrorT saNtfNotificationSubscribe(c
>       ntfsv_msg_t msg, *o_msg = NULL;
>       ntfsv_subscribe_req_t *send_param;
>       uint32_t timeout = NTFS_WAIT_TIME;
>-
>+      bool recovery_failed = false;
>       TRACE_ENTER();
>+
>       rc = getFilters(notificationFilterHandles, &filters, &ntfHandle,
>&client_hdl_rec);
>       if (rc != SA_AIS_OK) {
>               TRACE("getFilters failed");
>               goto done;
>       }
>
>+      /* Check NTF server availability */
>+      if ((rc = checkNtfServerState()) != SA_AIS_OK)
>+              goto done;
>+
>+      /* recovery */
>+      if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
>+              if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>+                      recovery_failed = true;
>+                      goto done;
>+              }
>+      }
>+
>       tmpHandle = ntfHandleGet(subscriptionId);
>       if (tmpHandle != 0) {
>               rc = SA_AIS_ERR_EXIST;
>               goto done;
>       }
>-      rc = subscriptionListAdd(ntfHandle, subscriptionId);
>+      rc = subscriptionListAdd(ntfHandle, subscriptionId, filters);
>       if (rc != SA_AIS_OK) {
>               goto done;
>       }
>@@ -1693,29 +2033,24 @@ SaAisErrorT saNtfNotificationSubscribe(c
>       send_param->client_id = client_hdl_rec->ntfs_client_id;
>       send_param->subscriptionId = subscriptionId;
>       send_param->f_rec = filters;
>-      /* Check whether NTFS is up or not */
>-      if (ntfa_cb.ntfs_up) {
>-              uint32_t rv;
>-              /* Send a sync MDS message to obtain a log stream id */
>-              rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>timeout);
>-              if (rv == NCSCC_RC_SUCCESS) {
>-                      osafassert(o_msg != NULL);
>-                      if (SA_AIS_OK == o_msg->info.api_resp_info.rc) {
>-                              TRACE_1("subscriptionId from server %u",
>-                                      o_msg-
>>info.api_resp_info.param.subscribe_rsp.subscriptionId);
>-                      } else {
>-                              rc = o_msg->info.api_resp_info.rc;
>-                              TRACE("Bad return status!!! rc = %d", rc);
>-                      }
>+
>+      uint32_t rv;
>+      /* Send a sync MDS message to obtain a log stream id */
>+      rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>+      if (rv == NCSCC_RC_SUCCESS) {
>+              osafassert(o_msg != NULL);
>+              if (SA_AIS_OK == o_msg->info.api_resp_info.rc) {
>+                      TRACE_1("subscriptionId from server %u",
>+                              o_msg-
>>info.api_resp_info.param.subscribe_rsp.subscriptionId);
>               } else {
>-                      if(rv == NCSCC_RC_INVALID_INPUT)
>-                              rc = SA_AIS_ERR_INVALID_PARAM;
>-                      else
>-                              rc = SA_AIS_ERR_TRY_AGAIN;
>+                      rc = o_msg->info.api_resp_info.rc;
>+                      TRACE("Bad return status!!! rc = %d", rc);
>               }
>       } else {
>-              TRACE_1("NTFS down");
>-              rc = SA_AIS_ERR_TRY_AGAIN;
>+              if(rv == NCSCC_RC_INVALID_INPUT)
>+                      rc = SA_AIS_ERR_INVALID_PARAM;
>+              else
>+                      rc = SA_AIS_ERR_TRY_AGAIN;
>       }
>
[Vu] Seems missing SA_AIS_ERR_TIMEOUT err code here. 
>       if (rc != SA_AIS_OK) {
>@@ -1724,6 +2059,7 @@ SaAisErrorT saNtfNotificationSubscribe(c
>       if (o_msg)
>               ntfa_msg_destroy(o_msg);
>       done:
>+
>       ncshm_give_hdl(ntfHandle);
>       if (notificationFilterHandles) {
>               if (notificationFilterHandles->attributeChangeFilterHandle)
>@@ -1737,6 +2073,13 @@ SaAisErrorT saNtfNotificationSubscribe(c
>               if (notificationFilterHandles->alarmFilterHandle)
>                       ncshm_give_hdl(notificationFilterHandles-
>>alarmFilterHandle);
>       }
>+      if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
>+              osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>+              ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
client_hdl_rec);
>+              osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>+              ntfa_shutdown(false);
>+      }
>+
>       TRACE_LEAVE();
>       return rc;
> }
>@@ -1995,6 +2338,7 @@ SaAisErrorT saNtfSecurityAlarmNotificati
>               rc = SA_AIS_ERR_INVALID_PARAM;
>               goto done;
>       }
>+
>       /* retrieve hdl rec */
>       hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA, ntfHandle);
>       if (hdl_rec == NULL) {
>@@ -2058,6 +2402,7 @@ SaAisErrorT saNtfPtrValAllocate(SaNtfNot
>               rc = SA_AIS_ERR_INVALID_PARAM;
>               goto done;
>       }
>+
>       notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>notificationHandle);
>       if (notification_hdl_rec == NULL) {
>               TRACE("ncshm_take_hdl notificationHandle failed");
>@@ -2098,6 +2443,7 @@ SaAisErrorT saNtfArrayValAllocate(SaNtfN
>               rc = SA_AIS_ERR_INVALID_PARAM;
>               goto done;
>       }
>+
>       notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>notificationHandle);
>       if (notification_hdl_rec == NULL) {
>               TRACE("ncshm_take_hdl notificationHandle failed");
>@@ -2228,6 +2574,7 @@ SaAisErrorT saNtfArrayValGet(SaNtfNotifi
>               rc = SA_AIS_ERR_INVALID_PARAM;
>               goto done;
>       }
>+
>       notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>notificationHandle);
>       if (notification_hdl_rec == NULL) {
>               TRACE("ncshm_take_hdl notificationHandle failed");
>@@ -2695,7 +3042,7 @@ SaAisErrorT saNtfNotificationFilterFree(
>       }
>
>       /* free the resources allocated by saNtf<ntfType>FilterAllocate */
>-      ntfa_filter_hdl_rec_destructor(filter_hdl_rec);
>+      ntfa_filter_destructor(filter_hdl_rec);
>
>     /** Delete the resources related to the notificationFilterHandle &
>      *  remove reference in the client.
>@@ -2729,7 +3076,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>       TRACE_ENTER();
>       SaAisErrorT rc = SA_AIS_ERR_NOT_EXIST;
>       SaNtfHandleT ntfHandle;
>-
>       ntfa_client_hdl_rec_t *client_hdl_rec;
>
>       ntfsv_msg_t msg, *o_msg = NULL;
>@@ -2737,7 +3083,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>       ntfsv_unsubscribe_req_t *send_param;
>       uint32_t timeout = NTFS_WAIT_TIME;
>
>-
>       ntfHandle = ntfHandleGet(subscriptionId);
>       if (ntfHandle == 0) {
>               TRACE_1("ntfHandleGet failed, subscription not exist");
>@@ -2753,9 +3098,16 @@ SaAisErrorT saNtfNotificationUnsubscribe
>               goto done;
>       }
>
>-      /**
>-         ** Populate a sync MDS message
>-         **/
>+      /* Check NTF server availability */
>+      if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>+              ncshm_give_hdl(ntfHandle);
>+              goto done;
>+      }
>+
>+      if (client_hdl_rec->valid) {
>+              /**
>+               ** Populate a sync MDS message
>+               **/
>               memset(&msg, 0, sizeof(ntfsv_msg_t));
>               msg.type = NTFSV_NTFA_API_MSG;
>               msg.info.api_info.type = NTFSV_UNSUBSCRIBE_REQ;
>@@ -2764,13 +3116,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>               send_param->client_id = client_hdl_rec->ntfs_client_id;
>               send_param->subscriptionId = subscriptionId;
>
>-              /* Check whether NTFS is up or not */
>-              if (!ntfa_cb.ntfs_up) {
>-                      TRACE_1("NTFS down");
>-                      rc = SA_AIS_ERR_TRY_AGAIN;
>-                      goto done_give_hdl;
>-              }
>-
>               /* Send a sync MDS message to obtain a log stream id */
>               rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>timeout);
>               if (rc != NCSCC_RC_SUCCESS) {
>@@ -2785,6 +3130,7 @@ SaAisErrorT saNtfNotificationUnsubscribe
>                       TRACE_1("Bad return status! rc = %d", rc);
>                       goto done_give_hdl;
>               }
>+      }
>               subscriberListItemRemove(subscriptionId);
>
>               /*Remove msg for subscriptionId from mailbox*/
>@@ -2815,6 +3161,19 @@ SaAisErrorT saNtfNotificationUnsubscribe
>  done_give_hdl:
>       if (o_msg)
>                ntfa_msg_destroy(o_msg);
>+
>+      if (!client_hdl_rec->valid) {
>+              if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>+                      if (rc == SA_AIS_ERR_BAD_HANDLE) {
>+                              ncshm_give_hdl(ntfHandle);
>+
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>+                              ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
>client_hdl_rec);
>+
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>+                              ntfa_shutdown(false);
>+                              goto done;
>+                      }
>+              }
>+      }
>       ncshm_give_hdl(ntfHandle);
>  done:
>       TRACE_LEAVE();
>@@ -2838,6 +3197,7 @@ SaAisErrorT saNtfNotificationReadInitial
>       ntfsv_reader_init_req_2_t *send_param;
>       uint32_t timeout = NTFS_WAIT_TIME;
>
>+      bool recovery_failed = false;
>       TRACE_ENTER();
>       if (notificationFilterHandles == NULL || readHandle == NULL) {
>               rc = SA_AIS_ERR_INVALID_PARAM;
>@@ -2872,6 +3232,18 @@ SaAisErrorT saNtfNotificationReadInitial
>               goto done_give_client_hdl;
>       }
>
>+      /* Check NTF server availability */
>+      if ((rc = checkNtfServerState()) != SA_AIS_OK)
>+              goto done_give_client_hdl;
>+
>+      /* recovery */
>+      if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
>+              if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>+                      recovery_failed = true;
>+                      goto done_give_client_hdl;
>+              }
>+      }
>+
>     /**
>      ** Populate a sync MDS message
>      **/
>@@ -2884,13 +3256,6 @@ SaAisErrorT saNtfNotificationReadInitial
>       send_param->head.searchCriteria = searchCriteria;
>       send_param->f_rec = filters;
>
>-      /* Check whether NTFS is up or not */
>-      if (!ntfa_cb.ntfs_up) {
>-              TRACE("NTFS down");
>-              rc = SA_AIS_ERR_TRY_AGAIN;
>-              goto done_give_client_hdl;
>-      }
>-
>       /* Send a sync MDS message to obtain a log stream id */
>       rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>       if (rc != NCSCC_RC_SUCCESS) {
>@@ -2925,12 +3290,16 @@ SaAisErrorT saNtfNotificationReadInitial
>       reader_hdl_rec->ntfHandle = ntfHandle;
>       /* Store the readerId returned from server */
>       reader_hdl_rec->reader_id = o_msg-
>>info.api_resp_info.param.reader_init_rsp.readerId;
>+      memset(&reader_hdl_rec->filters, 0, sizeof(ntfsv_filter_ptrs_t));
>+      ntfa_copy_ntf_filter_ptrs(&reader_hdl_rec->filters, &filters);
>+      reader_hdl_rec->searchCriteria = searchCriteria;
>     /**                  UnLock ntfa_CB            **/
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>
> done_give_client_hdl:
>       if (o_msg)
>               ntfa_msg_destroy(o_msg);
>+
>    if (client_hdl_rec)
>               ncshm_give_hdl(client_hdl_rec->local_hdl);
>       if (notificationFilterHandles) {
>@@ -2947,6 +3316,12 @@ done_give_client_hdl:
>       }
>
>       ncshm_give_hdl(notificationFilterHandles->alarmFilterHandle);
>+      if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
>+              osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>+              ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
client_hdl_rec);
>+              osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>+              ntfa_shutdown(false);
>+      }
> done:
>       TRACE_LEAVE();
>       return rc;
>@@ -2983,42 +3358,46 @@ SaAisErrorT saNtfNotificationReadFinaliz
>       }
>       TRACE_1("reader_hdl_rec = %u", reader_hdl_rec->reader_hdl);
>
>-    /**
>-     ** Populate a sync MDS message
>-     **/
>-      memset(&msg, 0, sizeof(ntfsv_msg_t));
>-      msg.type = NTFSV_NTFA_API_MSG;
>-      msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
>-      send_param = &msg.info.api_info.param.reader_finalize;
>-      send_param->client_id = client_hdl_rec->ntfs_client_id;
>-      send_param->readerId = reader_hdl_rec->reader_id;
>-
>-      /* Check whether NTFS is up or not */
>-      if (!ntfa_cb.ntfs_up) {
>-              TRACE("NTFS down");
>-              rc = SA_AIS_ERR_TRY_AGAIN;
>-              goto done_give_hdls;
>+      /* Check NTF server availability */
>+      if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>+              ncshm_give_hdl(client_hdl_rec->local_hdl);
>+              ncshm_give_hdl(readhandle);
>+              goto done;
>       }
>
>-      /* Send a sync MDS message */
>-      rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>-      if (rc != NCSCC_RC_SUCCESS) {
>-              rc = SA_AIS_ERR_TRY_AGAIN;
>-              goto done_give_hdls;
>+      if (client_hdl_rec->valid) {
>+              /**
>+               ** Populate a sync MDS message
>+               **/
>+              memset(&msg, 0, sizeof(ntfsv_msg_t));
>+              msg.type = NTFSV_NTFA_API_MSG;
>+              msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
>+              send_param = &msg.info.api_info.param.reader_finalize;
>+              send_param->client_id = client_hdl_rec->ntfs_client_id;
>+              send_param->readerId = reader_hdl_rec->reader_id;
>+
>+
>+              /* Send a sync MDS message */
>+              rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>timeout);
>+              if (rc != NCSCC_RC_SUCCESS) {
>+                      rc = SA_AIS_ERR_TRY_AGAIN;
>+                      goto done_give_hdls;
>+              }
>+
>+              osafassert(o_msg != NULL);
>+              if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
>+                      rc = o_msg->info.api_resp_info.rc;
>+                      TRACE("Bad return status!!! rc = %d", rc);
>+                      goto done_give_hdls;
>+              }
>+
>+              if (o_msg->info.api_resp_info.type !=
>NTFSV_READER_FINALIZE_RSP) {
>+                      TRACE("msg type (%d) failed", (int)o_msg-
>>info.api_resp_info.type);
>+                      rc = SA_AIS_ERR_LIBRARY;
>+                      goto done_give_hdls;
>+              }
>       }
>
>-      osafassert(o_msg != NULL);
>-      if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
>-              rc = o_msg->info.api_resp_info.rc;
>-              TRACE("Bad return status!!! rc = %d", rc);
>-              goto done_give_hdls;
>-      }
>-
>-      if (o_msg->info.api_resp_info.type != NTFSV_READER_FINALIZE_RSP) {
>-              TRACE("msg type (%d) failed", (int)o_msg-
>>info.api_resp_info.type);
>-              rc = SA_AIS_ERR_LIBRARY;
>-              goto done_give_hdls;
>-      }
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>       oas_rc = ntfa_reader_hdl_rec_del(&client_hdl_rec->reader_list,
>reader_hdl_rec);
>       if (oas_rc != NCSCC_RC_SUCCESS) {
>@@ -3029,6 +3408,20 @@ SaAisErrorT saNtfNotificationReadFinaliz
>  done_give_hdls:
>       if (o_msg)
>                ntfa_msg_destroy(o_msg);
>+
>+      if (!client_hdl_rec->valid) {
>+              if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>+                      if (rc == SA_AIS_ERR_BAD_HANDLE) {
>+                              ncshm_give_hdl(client_hdl_rec->local_hdl);
>+                              ncshm_give_hdl(readhandle);
>+
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>+                              ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
>client_hdl_rec);
>+
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>+                              ntfa_shutdown(false);
>+                              goto done;
>+                      }
>+              }
>+      }
>       ncshm_give_hdl(client_hdl_rec->local_hdl);
>  done_give_read_hdl:
>       ncshm_give_hdl(readhandle);
>@@ -3082,6 +3475,23 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>       }
>       TRACE_1("reader_hdl_rec = %u", reader_hdl_rec->reader_hdl);
>
>+      /* Check NTF server availability */
>+      if ((rc = checkNtfServerState()) != SA_AIS_OK)
>+              goto done_give_hdls;
>+
>+      if (!client_hdl_rec->valid) {
>+              if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>+                      ncshm_give_hdl(client_hdl_rec->local_hdl);
>+                      ncshm_give_hdl(readHandle);
>+                      if (rc == SA_AIS_ERR_BAD_HANDLE) {
>+
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>+                              ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
>client_hdl_rec);
>+
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>+                              ntfa_shutdown(false);
>+                      }
>+                      goto done;
>+              }
>+      }
>     /**
>      ** Populate a sync MDS message
>      **/
>@@ -3092,12 +3502,6 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>       send_param->client_id = client_hdl_rec->ntfs_client_id;
>       send_param->readerId = reader_hdl_rec->reader_id;
>       send_param->searchDirection = searchDirection;
>-      /* Check whether NTFS is up or not */
>-      if (!ntfa_cb.ntfs_up) {
>-              TRACE("NTFS down");
>-              rc = SA_AIS_ERR_TRY_AGAIN;
>-              goto done_give_hdls;
>-      }
>
>       do {
>               /* Send a sync MDS message */
>diff --git a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>--- a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>+++ b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>@@ -369,7 +369,8 @@ static uint32_t ntfa_mds_svc_evt(struct
>                       TRACE("NTFS down");
>                       pthread_mutex_lock(&ntfa_cb.cb_lock);
>                       memset(&ntfa_cb.ntfs_mds_dest, 0,
>sizeof(MDS_DEST));
>-                      ntfa_cb.ntfs_up = 0;
>+                      ntfa_update_ntfsv_state(mds_cb_info-
>>info.svc_evt.i_change == NCSMDS_NO_ACTIVE ?
>+                                      NTFA_NTFSV_NO_ACTIVE :
>NTFA_NTFSV_DOWN);
>                       pthread_mutex_unlock(&ntfa_cb.cb_lock);
>               }
>               break;
>@@ -382,7 +383,8 @@ static uint32_t ntfa_mds_svc_evt(struct
>                       TRACE_2("MSG from NTFS NCSMDS_NEW_ACTIVE/UP");
>                       pthread_mutex_lock(&ntfa_cb.cb_lock);
>                       ntfa_cb.ntfs_mds_dest = mds_cb_info-
>>info.svc_evt.i_dest;
>-                      ntfa_cb.ntfs_up = 1;
>+                      ntfa_update_ntfsv_state(mds_cb_info-
>>info.svc_evt.i_change == NCSMDS_NEW_ACTIVE ?
>+                                      NTFA_NTFSV_NEW_ACTIVE :
>NTFA_NTFSV_UP);
>                       if (ntfa_cb.ntfs_sync_awaited) {
>                               /* signal waiting thread */
>                               m_NCS_SEL_OBJ_IND(&ntfa_cb.ntfs_sync_sel);
>diff --git a/osaf/libs/agents/saf/ntfa/ntfa_util.c
>b/osaf/libs/agents/saf/ntfa/ntfa_util.c
>--- a/osaf/libs/agents/saf/ntfa/ntfa_util.c
>+++ b/osaf/libs/agents/saf/ntfa/ntfa_util.c
>@@ -49,7 +49,7 @@ static unsigned int ntfa_create(void)
>       }
>
>       /* Block and wait for indication from MDS meaning NTFS is up */
>-      osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_sync_sel),
>30000);
>+      osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_sync_sel),
>10000);
>
>       pthread_mutex_lock(&ntfa_cb.cb_lock);
>       ntfa_cb.ntfs_sync_awaited = 0;
>@@ -108,7 +108,25 @@ static bool ntfa_clear_mbx(NCSCONTEXT ar
>       }
>       return true;
> }
>+/****************************************************************
>************
>+  Name          : ntfa_notification_list_del
>
>+  Description   : This routine free element allocated in list of
notification
>+
>+  Arguments     : pointer to the list of notification records anchor.
>+
>+  Return Values : None
>+
>+  Notes         :
>+*****************************************************************
>*************/
>+static void ntfa_notification_list_del(ntfa_notification_hdl_rec_t
**plstr_hdl)
>+{
>+      ntfa_notification_hdl_rec_t *lstr_hdl = *plstr_hdl;
>+      while (lstr_hdl != NULL) {
>+              ntfa_notification_destructor(lstr_hdl);
>+              lstr_hdl = lstr_hdl->next;
>+      }
>+}
>
>/*****************************************************************
>***********
>   Name          : ntfa_notification_hdl_rec_list_del
>
>@@ -130,6 +148,70 @@ static void ntfa_notification_hdl_rec_li
>               lstr_hdl = NULL;
>       }
> }
>+/****************************************************************
>************
>+  Name          : ntfa_filter_list_del
>+
>+  Description   : This routine free element allocated in list of filter
>+
>+  Arguments     : pointer to the list of filters records anchor.
>+
>+  Return Values : None
>+
>+  Notes         :
>+*****************************************************************
>*************/
>+static void ntfa_filter_list_del(ntfa_filter_hdl_rec_t **plstr_hdl)
>+{
>+      ntfa_filter_hdl_rec_t *lstr_hdl = *plstr_hdl;
>+
>+      while (lstr_hdl != NULL) {
>+              ntfa_filter_destructor(lstr_hdl);
>+              lstr_hdl = lstr_hdl->next;
>+      }
>+}
>+/****************************************************************
>************
>+  Name          : ntfa_filter_hdl_rec_list_del
>+
>+  Description   : This routine deletes a list of allocated filters
>+
>+  Arguments     : pointer to the list of filters records anchor.
>+
>+  Return Values : None
>+
>+  Notes         :
>+*****************************************************************
>*************/
>+static void ntfa_filter_hdl_rec_list_del(ntfa_filter_hdl_rec_t
**plstr_hdl)
>+{
>+      ntfa_filter_hdl_rec_t *lstr_hdl;
>+
>+      while ((lstr_hdl = *plstr_hdl) != NULL) {
>+              *plstr_hdl = lstr_hdl->next;
>+              ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, lstr_hdl-
>>filter_hdl);
>+              free(lstr_hdl);
>+              lstr_hdl = NULL;
>+      }
>+}
>+/****************************************************************
>************
>+  Name          : ntfa_reader_hdl_rec_list_del
>+
>+  Description   : This routine deletes a list of allocated readers
>+
>+  Arguments     : pointer to the list of readers records anchor.
>+
>+  Return Values : None
>+
>+  Notes         :
>+*****************************************************************
>*************/
>+static void ntfa_reader_hdl_rec_list_del(ntfa_reader_hdl_rec_t
**plstr_hdl)
>+{
>+      ntfa_reader_hdl_rec_t *lstr_hdl;
>+      while ((lstr_hdl = *plstr_hdl) != NULL) {
>+              *plstr_hdl = lstr_hdl->next;
>+              ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, lstr_hdl-
>>reader_hdl);
>+              ntfa_del_ntf_filter_ptrs(&lstr_hdl->filters);
>+              free(lstr_hdl);
>+              lstr_hdl = NULL;
>+      }
>+}
>
> static SaAisErrorT ntfa_alloc_callback_notification(SaNtfNotificationsT
>*notification,
>                                                   ntfsv_send_not_req_t
>*not_cbk, ntfa_client_hdl_rec_t *hdl_rec)
>@@ -407,6 +489,10 @@ static SaAisErrorT ntfa_hdl_cbk_rec_prc(
>                       free(cbk_info-
>>param.discarded_cbk.discardedNotificationIdentifiers);
>               }
>               break;
>+      case NTFSV_DUMMY_CALLBACK:
>+              TRACE("Do nothing with dummy callback, just return OK");
>+              rc = SA_AIS_OK;
>+              break;
>       default:
>               TRACE("unsupported callback type: %d", cbk_info->type);
>               rc = SA_AIS_ERR_LIBRARY;
>@@ -544,20 +630,21 @@ unsigned int ntfa_startup(void)
>  *
>  * @return unsigned int
>  */
>-unsigned int ntfa_shutdown(void)
>+unsigned int ntfa_shutdown(bool forced)
> {
>       unsigned int rc = NCSCC_RC_SUCCESS;
>
>-      TRACE_ENTER2("ntfa_use_count: %u", ntfa_use_count);
>+      TRACE_ENTER2("ntfa_use_count: %u, forced: %u", ntfa_use_count,
>forced);
>       pthread_mutex_lock(&ntfa_lock);
>
>-      if (ntfa_use_count > 1) {
>-              /* Users still exist, just decrement the use count */
>-              ntfa_use_count--;
>-      } else if (ntfa_use_count == 1) {
>+      if ((forced && (ntfa_use_count > 0)) || (ntfa_use_count == 1)) {
>               ntfa_destroy();
>               rc = ncs_agents_shutdown();
>               ntfa_use_count = 0;
>+              ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_NONE;
>+      } else if (ntfa_use_count > 1) {
>+              /* Users still exist, just decrement the use count */
>+              ntfa_use_count--;
>       }
>
>       pthread_mutex_unlock(&ntfa_lock);
>@@ -633,7 +720,27 @@ ntfa_client_hdl_rec_t *ntfa_find_hdl_rec
>
>       return NULL;
> }
>+/****************************************************************
>************
>+  Name          : ntfa_subscriber_list_del
>
>+  Description   : This routine deletes a list of allocated subscribers
>+
>+  Arguments     : pointer to the list of subscribers records anchor.
>+
>+  Return Values : None
>+
>+  Notes         :
>+*****************************************************************
>*************/
>+void ntfa_subscriber_list_del()
>+{
>+      ntfa_subscriber_list_t *listPtr = subscriberNoList;
>+      while (listPtr != NULL) {
>+              ntfa_subscriber_list_t* tmpSub = listPtr;
>+              listPtr = listPtr->next;
>+              free(tmpSub);
>+      }
>+      subscriberNoList = NULL;
>+}
>
>/*****************************************************************
>***********
>   Name          : ntfa_hdl_list_del
>
>@@ -651,15 +758,24 @@ void ntfa_hdl_list_del(ntfa_client_hdl_r
>
>       while ((client_hdl = *p_client_hdl) != NULL) {
>               *p_client_hdl = client_hdl->next;
>+              m_NCS_IPC_DETACH(&client_hdl->mbx, ntfa_clear_mbx, NULL);
>+              m_NCS_IPC_RELEASE(&client_hdl->mbx, NULL);
>               ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, client_hdl-
>>local_hdl);
>       /** clean up the channel records for this ntfa-client
>          **/
>+              ntfa_notification_list_del(&client_hdl->notification_list);
>               ntfa_notification_hdl_rec_list_del(&client_hdl-
>>notification_list);
>+
>+              ntfa_filter_list_del(&client_hdl->filter_list);
>+              ntfa_filter_hdl_rec_list_del(&client_hdl->filter_list);
>+
>+              ntfa_reader_hdl_rec_list_del(&client_hdl->reader_list);
>       /** remove the association with hdl-mngr
>          **/
>               free(client_hdl);
>               client_hdl = 0;
>       }
>+      ntfa_subscriber_list_del();
> }
>
>
>/*****************************************************************
>***********
>@@ -757,10 +873,80 @@ uint32_t ntfa_filter_hdl_rec_del(ntfa_fi
>       TRACE("The node couldn't be deleted");
>       return NCSCC_RC_FAILURE;
> }
>+/****************************************************************
>************
>+  Name          : ntfa_hdl_rec_force_del
>
>+  Description   : This routine deletes all memory allocated to client.
>+
>+  Arguments     : NTFA_CLIENT_HDL_REC **list_head
>+                NTFA_CLIENT_HDL_REC *rm_node
>+
>+  Return Values : None
>+
>+  Notes         :
>+*****************************************************************
>*************/
>+void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **list_head,
>ntfa_client_hdl_rec_t *rm_node)
>+{
>+      ntfa_client_hdl_rec_t *list_iter = *list_head;
>+      TRACE_ENTER();
>+      /* First remove the rm_node out of the list of client */
>+      if (list_iter == rm_node)
>+              *list_head = rm_node->next;
>+      else {
>+              while (list_iter) {
>+                      if (list_iter->next == rm_node) {
>+                              list_iter->next = rm_node->next;
>+                              break;
>+                      }
>+                      list_iter = list_iter->next;
>+              }
>+      }
>+      /* Release all msgs in mailbox */
>+      ntfsv_msg_t *cbk_msg;
>+      while((cbk_msg =
>(ntfsv_msg_t*)m_NCS_IPC_NON_BLK_RECEIVE(&rm_node->mbx, cbk_msg))
>+              != NULL) {
>+              ntfa_msg_destroy(cbk_msg);
>+      }
>+      /* delete subscriber of this client out of the subcriberNoList*/
>+      ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
>+      while (subscriber_hdl != NULL) {
>+              ntfa_subscriber_list_t *rm_subscriber = subscriber_hdl;
>+              subscriber_hdl = subscriber_hdl->next;
>+              if (rm_node->local_hdl == rm_subscriber-
>>subscriberListNtfHandle) {
>+                      if (rm_subscriber->next != NULL) {
>+                              rm_subscriber->next->prev = rm_subscriber-
>>prev;
>+                      }
>+
>+                      if (rm_subscriber->prev != NULL) {
>+                              rm_subscriber->prev->next = rm_subscriber-
>>next;
>+                      } else {
>+                              if (rm_subscriber->next != NULL)
>+                                      subscriberNoList = rm_subscriber-
>>next;
>+                              else
>+                                      subscriberNoList = NULL;
>+                      }
>+                      ntfa_del_ntf_filter_ptrs(&rm_subscriber->filters);
>+                      free(rm_subscriber);
>+              }
>+      }
>+      /* Now delete client */
>+      m_NCS_IPC_DETACH(&rm_node->mbx, ntfa_clear_mbx, NULL);
>+      m_NCS_IPC_RELEASE(&rm_node->mbx, NULL);
>+      ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node->local_hdl);
>+      ntfa_notification_list_del(&rm_node->notification_list);
>+      ntfa_notification_hdl_rec_list_del(&rm_node->notification_list);
>+
>+      ntfa_filter_list_del(&rm_node->filter_list);
>+      ntfa_filter_hdl_rec_list_del(&rm_node->filter_list);
>+
>+      ntfa_reader_hdl_rec_list_del(&rm_node->reader_list);
>+      free(rm_node);
>+
>+      TRACE_LEAVE();
>+}
>
>/*****************************************************************
>***********
>   Name          : ntfa_hdl_rec_del
>-
>+
>   Description   : This routine deletes the a client handle record from
>                   a list of client hdl records.
>
>@@ -962,7 +1148,7 @@ ntfa_client_hdl_rec_t *ntfa_hdl_rec_add(
>     /** Associate with the client_id obtained from NTFS
>      **/
>       rec->ntfs_client_id = client_id;
>-
>+      rec->valid = true;
>     /** Initialize and attach the IPC/Priority queue
>      **/
>
>@@ -1063,7 +1249,7 @@ static void logtrace_init_constructor(vo
>  *
>  * @param instance
>  */
>-void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t *instance)
>+void ntfa_notification_destructor(ntfa_notification_hdl_rec_t *instance)
> {
>       ntfa_notification_hdl_rec_t *notificationInstance = instance;
>
>@@ -1111,7 +1297,7 @@ void ntfa_hdl_rec_destructor(ntfa_notifi
>  *
>  * @param instance
>  */
>-void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t *filter_rec)
>+void ntfa_filter_destructor(ntfa_filter_hdl_rec_t *filter_rec)
> {
>       switch (filter_rec->ntfType) {
>       case SA_NTF_TYPE_OBJECT_CREATE_DELETE:
>@@ -1202,6 +1388,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>          **/
>               ncshm_give_hdl(rm_node->reader_hdl);
>               ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node-
>>reader_hdl);
>+              ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>               free(rm_node);
>               return NCSCC_RC_SUCCESS;
>       } else {                /* find the rec */
>@@ -1213,6 +1400,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>                  **/
>                               ncshm_give_hdl(rm_node->reader_hdl);
>                               ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA,
>rm_node->reader_hdl);
>+                              ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>                               free(rm_node);
>                               return NCSCC_RC_SUCCESS;
>                       }
>@@ -1258,3 +1446,258 @@ void ntfa_add_to_async_cbk_msg_list(ntfs
>
>       TRACE_LEAVE();
> }
>+
>+/****************************************************************
>************
>+  Name          : ntfa_notify_handle_invalid
>+
>+  Description   : This routine sends a dummy callback msg to client's
mailbox
>+                so that the client polls in and calls saNtfDispatch.
>+  Arguments     :
>+
>+  Return Values : None
>+
>+  Notes         : None
>+*****************************************************************
>*************/
>+void ntfa_notify_handle_invalid() {
>+      ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
>+      TRACE_ENTER();
>+      while (client_hdl != NULL) {
>+              /* Only applicable for subscriber */
>+              if (client_hdl->reg_cbk.saNtfNotificationCallback != NULL ||
>+
client_hdl->reg_cbk.saNtfNotificationDiscardedCallback
>!= NULL) {
>+                      /* Create a dummy msg */
>+                      ntfsv_msg_t *msg = malloc(sizeof(ntfsv_msg_t));
>+                      memset(msg, 0, sizeof(ntfsv_msg_t));
>+                      msg->info.cbk_info.type = NTFSV_DUMMY_CALLBACK;
>+                      /* Send dummy msg to client mailbox */
>+                      if (m_NCS_IPC_SEND(&client_hdl->mbx, msg,
>MDS_SEND_PRIORITY_HIGH) != NCSCC_RC_SUCCESS) {
>+                              TRACE_1("m_NCS_IPC_SEND Failed to
>client(id:%u)", client_hdl->ntfs_client_id);
>+                              ntfa_msg_destroy(msg);
>+                      }
>+              }
>+
>+              client_hdl = client_hdl->next;
>+      }
>+      TRACE_LEAVE();
>+}
>+/****************************************************************
>************
>+  Name          : ntfa_update_ntfsv_state
>+
>+  Description   : Update current NTF Server state by the @changedState
>indicated
>+                by MDS event
>+  Arguments     : @changedState [IN]: state to be changed of NTF Server
>+
>+  Return Values : None
>+
>+  Notes         : None
>+*****************************************************************
>*************/
>+void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState)
>+{
>+      TRACE_ENTER();
>+      TRACE_1("Current state: %u, Changed state: %u",
>ntfa_cb.ntfa_ntfsv_state,
>+
>                       changedState);
>+
>+      ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
>+
>+      switch (ntfa_cb.ntfa_ntfsv_state){
>+      case NTFA_NTFSV_NONE:
>+              ntfa_cb.ntfa_ntfsv_state = changedState;
>+              break;
>+      case NTFA_NTFSV_DOWN:
>+              if (changedState == NTFA_NTFSV_NEW_ACTIVE ||
>+                      changedState == NTFA_NTFSV_UP) {
>+                      TRACE("Active NTF server has been restarted");
>+                      ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_UP;
>+                      ntfa_notify_handle_invalid();
>+              } else
>+                      TRACE("Unexpected state changes");
>+              break;
>+      case NTFA_NTFSV_NO_ACTIVE:
>+              if (changedState == NTFA_NTFSV_NEW_ACTIVE) {
>+                      TRACE("Standby NTF server becomes new Active");
>+                      /* NTF server is functioning normally */
>+                      ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_UP;
>+              } else if (changedState == NTFA_NTFSV_DOWN) {
>+                      TRACE("Active NTF Server is Down");
>+                      ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_DOWN;
>+                      /* Mark all client handles are invalid */
>+                      while (client_hdl != NULL) {
>+                              client_hdl->valid = false;
>+                              client_hdl = client_hdl->next;
>+                      }
>+              }
>+              break;
>+      case NTFA_NTFSV_NEW_ACTIVE:
>+              TRACE("Unknown");
>+              break;
>+      case NTFA_NTFSV_UP:
>+              if (changedState == NTFA_NTFSV_NO_ACTIVE) {
>+                      TRACE("Active NTF server temporarily unavailable");
>+                      /* Failover/Switchover is happening
>+                       * Any API calls result in TRY_AGAIN
>+                       */
>+                      ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_NO_ACTIVE;
>+              } else
>+                      TRACE("Unexpected state changes");
>+              break;
>+      default:
>+              osafassert(false);
>+      }
>+
>+      TRACE_LEAVE();
>+}
>+/****************************************************************
>************
>+  Name          : ntfa_copy_ntf_filter_ptrs
>+
>+  Description   : Copy a list of filter from pSrc to pDes
>+
>+  Arguments     : pDes* [OUT]: list of outcome filter
>+                pSrc* [IN]: list of input filter
>+
>+  Return Values : SA_AIS_OK if succeed, other values as failed
>+
>+  Notes         : None
>+*****************************************************************
>*************/
>+SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
>+                                                              const
>ntfsv_filter_ptrs_t* pSrc) {
>+      SaAisErrorT rc = SA_AIS_OK;
>+      SaNtfNotificationFilterHeaderT *des_header;
>+      SaNtfNotificationFilterHeaderT *src_header;
>+      TRACE_ENTER();
>+
>+      if (pSrc->alarm_filter) {
>+              pDes->alarm_filter = calloc(1,
>sizeof(SaNtfAlarmNotificationFilterT));
>+              des_header =
&(pDes->alarm_filter->notificationFilterHeader);
>+              src_header =
&(pSrc->alarm_filter->notificationFilterHeader);
>+              if ((rc = ntfsv_filter_header_alloc(des_header, src_header-
>>numEventTypes,
>+
>               src_header->numNotificationObjects,
>+
>               src_header->numNotifyingObjects,
>+
>               src_header->numNotificationClassIds)) != SA_AIS_OK)
>+                      goto done;
>+              if ((rc = ntfsv_filter_alarm_alloc(pDes->alarm_filter,
>+
>               pSrc->alarm_filter->numProbableCauses,
>+
>               pSrc->alarm_filter->numPerceivedSeverities,
>+
>               pSrc->alarm_filter->numTrends)) != SA_AIS_OK)
>+                      goto done;
>+              if ((rc = ntfsv_copy_ntf_filter_alarm(pDes->alarm_filter,
>+
>                       pSrc->alarm_filter)) != SA_AIS_OK)
>+                      goto done;
>+      }
>+
>+      if (pSrc->sec_al_filter) {
>+              pDes->sec_al_filter = calloc(1,
>sizeof(SaNtfSecurityAlarmNotificationFilterT));
>+              des_header =
&(pDes->sec_al_filter->notificationFilterHeader);
>+              src_header =
&(pSrc->sec_al_filter->notificationFilterHeader);
>+              if ((rc = ntfsv_filter_header_alloc(des_header, src_header-
>>numEventTypes,
>+
>               src_header->numNotificationObjects,
>+
>               src_header->numNotifyingObjects,
>+
>               src_header->numNotificationClassIds)) != SA_AIS_OK)
>+                      goto done;
>+              if ((rc = ntfsv_filter_sec_alarm_alloc(pDes->sec_al_filter,
>+
>               pSrc->sec_al_filter->numProbableCauses,
>+
>               pSrc->sec_al_filter->numSeverities,
>+
>               pSrc->sec_al_filter->numSecurityAlarmDetectors,
>+
>               pSrc->sec_al_filter->numServiceUsers,
>+
>               pSrc->sec_al_filter->numServiceProviders)) != SA_AIS_OK)
>+                      goto done;
>+              if ((rc =
ntfsv_copy_ntf_filter_sec_alarm(pDes->sec_al_filter,
>+
>                       pSrc->sec_al_filter)) != SA_AIS_OK)
>+                      goto done;
>+      }
>+
>+      if (pSrc->sta_ch_filter) {
>+              pDes->sta_ch_filter = calloc(1,
>sizeof(SaNtfStateChangeNotificationFilterT));
>+              des_header =
&(pDes->sta_ch_filter->notificationFilterHeader);
>+              src_header =
&(pSrc->sta_ch_filter->notificationFilterHeader);
>+              if ((rc = ntfsv_filter_header_alloc(des_header, src_header-
>>numEventTypes,
>+
>               src_header->numNotificationObjects,
>+
>               src_header->numNotifyingObjects,
>+
>               src_header->numNotificationClassIds)) != SA_AIS_OK)
>+                      goto done;
>+              if ((rc = ntfsv_filter_state_ch_alloc(pDes->sta_ch_filter,
>+
>               pSrc->sta_ch_filter->numSourceIndicators,
>+
>               pSrc->sta_ch_filter->numStateChanges)) != SA_AIS_OK)
>+                      goto done;
>+              if ((rc =
ntfsv_copy_ntf_filter_state_ch(pDes->sta_ch_filter,
>+
>                       pSrc->sta_ch_filter)) != SA_AIS_OK)
>+                      goto done;
>+      }
>+
>+      if (pSrc->obj_cr_del_filter) {
>+              pDes->obj_cr_del_filter = calloc(1,
>sizeof(SaNtfObjectCreateDeleteNotificationFilterT));
>+              des_header = &(pDes->obj_cr_del_filter-
>>notificationFilterHeader);
>+              src_header = &(pSrc->obj_cr_del_filter-
>>notificationFilterHeader);
>+              if ((rc = ntfsv_filter_header_alloc(des_header, src_header-
>>numEventTypes,
>+
>               src_header->numNotificationObjects,
>+
>               src_header->numNotifyingObjects,
>+
>               src_header->numNotificationClassIds)) != SA_AIS_OK)
>+                      goto done;
>+              if ((rc =
ntfsv_filter_obj_cr_del_alloc(pDes->obj_cr_del_filter,
>+
>               pSrc->obj_cr_del_filter->numSourceIndicators)) != SA_AIS_OK)
>+                      goto done;
>+              if ((rc = ntfsv_copy_ntf_filter_obj_cr_del(pDes-
>>obj_cr_del_filter,
>+
>                       pSrc->obj_cr_del_filter)) != SA_AIS_OK)
>+                      goto done;
>+      }
>+
>+      if (pSrc->att_ch_filter) {
>+              pDes->att_ch_filter = calloc(1,
>sizeof(SaNtfAttributeChangeNotificationFilterT));
>+              des_header =
&(pDes->att_ch_filter->notificationFilterHeader);
>+              src_header =
&(pSrc->att_ch_filter->notificationFilterHeader);
>+              if ((rc = ntfsv_filter_header_alloc(des_header, src_header-
>>numEventTypes,
>+
>               src_header->numNotificationObjects,
>+
>               src_header->numNotifyingObjects,
>+
>               src_header->numNotificationClassIds)) != SA_AIS_OK)
>+                      goto done;
>+              if ((rc =
ntfsv_filter_attr_change_alloc(pDes->att_ch_filter,
>+
>               pSrc->att_ch_filter->numSourceIndicators)) != SA_AIS_OK)
>+                      goto done;
>+              if ((rc = ntfsv_copy_ntf_filter_attr_ch(pDes->att_ch_filter,
>+
>                       pSrc->att_ch_filter)) != SA_AIS_OK)
>+                      goto done;
>+      }
>+done:
>+      TRACE_LEAVE();
>+      return rc;
>+}
>+
>+/****************************************************************
>************
>+  Name          : ntfa_del_ntf_filter_ptrs
>+
>+  Description   : Delete the filter pointers
>+
>+  Arguments     : filter_ptrs* [IN/OUT]: filter pointers
>+
>+  Return Values : SA_AIS_OK if succeed, other values as failed
>+
>+  Notes         : None
>+*****************************************************************
>*************/
>+SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t* filter_ptrs)
>+{
>+      SaAisErrorT rc = SA_AIS_OK;
>+      if (filter_ptrs->alarm_filter) {
>+              ntfsv_filter_alarm_free(filter_ptrs->alarm_filter, true);
>+              free(filter_ptrs->alarm_filter);
>+      }
>+
>+      if (filter_ptrs->sec_al_filter) {
>+              ntfsv_filter_sec_alarm_free(filter_ptrs->sec_al_filter,
true);
>+              free(filter_ptrs->sec_al_filter);
>+      }
>+
>+      if (filter_ptrs->sta_ch_filter) {
>+              ntfsv_filter_state_ch_free(filter_ptrs->sta_ch_filter,
true);
>+              free(filter_ptrs->sta_ch_filter);
>+      }
>+
>+      if (filter_ptrs->obj_cr_del_filter) {
>+              ntfsv_filter_obj_cr_del_free(filter_ptrs->obj_cr_del_filter,
true);
>+              free(filter_ptrs->obj_cr_del_filter);
>+      }
>+
>+      if (filter_ptrs->att_ch_filter) {
>+              ntfsv_filter_attr_ch_free(filter_ptrs->att_ch_filter, true);
>+              free(filter_ptrs->att_ch_filter);
>+      }
>+      return rc;
>+}


------------------------------------------------------------------------------
Site24x7 APM Insight: Get Deep Visibility into Application Performance
APM + Mobile APM + RUM: Monitor 3 App instances at just $35/Month
Monitor end-to-end web transactions and take corrective actions now
Troubleshoot faster and improve end-user experience. Signup Now!
http://pubads.g.doubleclick.net/gampad/clk?id=272487151&iu=/4140
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel

Reply via email to