Hi Minh,

I see that you are using mutexes inside the checkNtfServerState(). I don't 
think this is a good solution since the same mutex  is used directly in the 
function calling checkNtfServerState(). The mutex usage in the 
checkNtfServerState() is hidden and there is a risk that this function may be 
placed within a protected area in the calling function. It is better to write a 
note in the checkNtfServerState() function head clearly telling that this 
function is not thread safe and has to be protected with the cb_lock mutex and 
then use the mutex in the calling function.

Also I found a lot more unprotected usage of the global ntfa_cb structure but 
that is not directly related to the resilience update e.g a lot of functions 
taking a pointer to this structure. I think a ticket for this should be written 
now. The fix for checkNtfServerState() will just make sure that the resilience 
patch don't add even more thread related issues.

Thanks
Lennart

> -----Original Message-----
> From: Minh Hon Chau [mailto:minh.c...@dektech.com.au]
> Sent: den 14 mars 2016 03:08
> To: Lennart Lund; praveen.malv...@oracle.com; Vu Minh Nguyen; Minh
> Chau H
> Cc: opensaf-devel@lists.sourceforge.net
> Subject: [PATCH 2 of 5] NTF: Add support cloud resilience for NTF Agent
> [#1180] V3
> 
>  osaf/libs/agents/saf/ntfa/ntfa.h      |   31 +-
>  osaf/libs/agents/saf/ntfa/ntfa_api.c  |  702
> +++++++++++++++++++++++++++------
>  osaf/libs/agents/saf/ntfa/ntfa_mds.c  |   14 +-
>  osaf/libs/agents/saf/ntfa/ntfa_util.c |  465 +++++++++++++++++++++-
>  4 files changed, 1057 insertions(+), 155 deletions(-)
> 
> 
> The patch contains support for cloud resilience feature
> in NTF Agent code. Please refer README.HYDRA for content
> of the changes
> 
> diff --git a/osaf/libs/agents/saf/ntfa/ntfa.h
> b/osaf/libs/agents/saf/ntfa/ntfa.h
> --- a/osaf/libs/agents/saf/ntfa/ntfa.h
> +++ b/osaf/libs/agents/saf/ntfa/ntfa.h
> @@ -91,6 +91,7 @@ typedef struct ntfa_filter_hdl_rec {
>  typedef struct subscriberList {
>       SaNtfHandleT subscriberListNtfHandle;
>       SaNtfSubscriptionIdT subscriberListSubscriptionId;
> +     ntfsv_filter_ptrs_t filters; /* remember the filters used by this
> subscriber */
>       struct subscriberList *prev;
>       struct subscriberList *next;
>  } ntfa_subscriber_list_t;
> @@ -100,6 +101,10 @@ typedef struct ntfa_reader_hdl_rec {
>       unsigned int reader_id; /* handle value returned by NTFS
> for this client */
>       SaNtfHandleT ntfHandle;
>       unsigned int reader_hdl;        /* READER handle from handle
> mgr */
> +
> +     ntfsv_filter_ptrs_t filters; /* remember the filters used by this
> reader */
> +     SaNtfSearchCriteriaT searchCriteria; /* remember the
> searchCriteria for recovery */
> +
>       struct ntfa_reader_hdl_rec *next;       /* next pointer for
> the list in ntfa_cb_t */
>       struct ntfa_client_hdl_rec *parent_hdl; /* Back Pointer to
> the client instantiation */
>  } ntfa_reader_hdl_rec_t;
> @@ -114,24 +119,35 @@ typedef struct ntfa_client_hdl_rec {
>       ntfa_reader_hdl_rec_t *reader_list;
>       SYSF_MBX mbx;           /* priority q mbx
> b/w MDS & Library */
>       struct ntfa_client_hdl_rec *next;       /* next pointer for
> the list in ntfa_cb_t */
> +     bool valid;             /* handle is valid if it's known by
> NTF server, used for headless hydra */
> +     SaVersionT version; /* the API version is being used by client,
> used for recover after headless */
>  } ntfa_client_hdl_rec_t;
> 
>  /*
>   * The NTFA control block is the master anchor structure for all NTFA
>   * instantiations within a process.
>   */
> +typedef enum {
> +     NTFA_NTFSV_NONE = 0,
> +     NTFA_NTFSV_DOWN,
> +     NTFA_NTFSV_NO_ACTIVE,
> +     NTFA_NTFSV_NEW_ACTIVE,
> +     NTFA_NTFSV_UP
> +}ntfa_ntfsv_state_t;
> +
>  typedef struct {
>       pthread_mutex_t cb_lock;        /* CB lock */
>       ntfa_client_hdl_rec_t *client_list;     /* NTFA client
> handle database */
>       ntfa_reader_hdl_rec_t *reader_list;
>       MDS_HDL mds_hdl;        /* MDS handle */
>       MDS_DEST ntfs_mds_dest; /* NTFS absolute/virtual address
> */
> -     int ntfs_up;            /* Indicate that MDS subscription
> -                              * is complete */
> +
>       /* NTFS NTFA sync params */
>       int ntfs_sync_awaited;
>       NCS_SEL_OBJ ntfs_sync_sel;
>       SaUint32T ntf_var_data_limit;   /* max allowed
> variableDataSize */
> +     /* NTF Server state */
> +     ntfa_ntfsv_state_t ntfa_ntfsv_state;
>  } ntfa_cb_t;
> 
>  /* ntfa_saf_api.c */
> @@ -149,7 +165,7 @@ extern void ntfsv_ntfa_evt_free(struct n
> 
>  /* ntfa_init.c */
>  extern unsigned int ntfa_startup(void);
> -extern unsigned int ntfa_shutdown(void);
> +extern unsigned int ntfa_shutdown(bool forced);
> 
>  /* ntfa_hdl.c */
>  extern SaAisErrorT ntfa_hdl_cbk_dispatch(ntfa_cb_t *,
> ntfa_client_hdl_rec_t *, SaDispatchFlagsT);
> @@ -159,6 +175,7 @@ extern ntfa_notification_hdl_rec_t *ntfa
>  extern ntfa_filter_hdl_rec_t
> *ntfa_filter_hdl_rec_add(ntfa_client_hdl_rec_t **hdl_rec);
>  extern void ntfa_hdl_list_del(ntfa_client_hdl_rec_t **);
>  extern uint32_t ntfa_hdl_rec_del(ntfa_client_hdl_rec_t **,
> ntfa_client_hdl_rec_t *);
> +extern void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **,
> ntfa_client_hdl_rec_t *);
>  extern uint32_t ntfa_notification_hdl_rec_del(ntfa_notification_hdl_rec_t
> **, ntfa_notification_hdl_rec_t *);
>  extern uint32_t ntfa_filter_hdl_rec_del(ntfa_filter_hdl_rec_t **,
> ntfa_filter_hdl_rec_t *);
>  extern bool ntfa_validate_ntfa_client_hdl(ntfa_cb_t *ntfa_cb,
> ntfa_client_hdl_rec_t *find_hdl_rec);
> @@ -166,11 +183,15 @@ extern bool ntfa_validate_ntfa_client_hd
>  /* ntfa_util.c */
>  extern ntfa_client_hdl_rec_t *ntfa_find_hdl_rec_by_client_id(ntfa_cb_t
> *ntfa_cb, uint32_t client_id);
>  extern void ntfa_msg_destroy(ntfsv_msg_t *msg);
> -extern void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t
> *instance);
> -extern void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t
> +extern void ntfa_notification_destructor(ntfa_notification_hdl_rec_t
> *instance);
> +extern void ntfa_filter_destructor(ntfa_filter_hdl_rec_t
> 
> *notificationFilterInstance);
>  extern ntfa_reader_hdl_rec_t
> *ntfa_reader_hdl_rec_add(ntfa_client_hdl_rec_t **hdl_rec);
>  extern uint32_t ntfa_reader_hdl_rec_del(ntfa_reader_hdl_rec_t **,
> ntfa_reader_hdl_rec_t *);
>  extern void ntfa_add_to_async_cbk_msg_list(ntfsv_msg_t ** head,
> ntfsv_msg_t * new_node);
>  extern uint32_t ntfa_ntfs_msg_proc(ntfa_cb_t *cb, ntfsv_msg_t
> *ntfsv_msg, MDS_SEND_PRIORITY_TYPE prio);
> +extern void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState);
> +extern SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
> +
>                       const ntfsv_filter_ptrs_t* pSrc);
> +extern SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t* 
> filter_ptrs);
>  #endif   /* !NTFA_H */
> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_api.c
> b/osaf/libs/agents/saf/ntfa/ntfa_api.c
> --- a/osaf/libs/agents/saf/ntfa/ntfa_api.c
> +++ b/osaf/libs/agents/saf/ntfa/ntfa_api.c
> @@ -34,11 +34,61 @@
>  /* The main controle block */
>  ntfa_cb_t ntfa_cb = {
>       .cb_lock = PTHREAD_MUTEX_INITIALIZER,
> +     .ntfa_ntfsv_state = NTFA_NTFSV_NONE,
>  };
> 
>  /* list of subscriptions for this process */
>  ntfa_subscriber_list_t *subscriberNoList = NULL;
> 
> +/*
> + * @Brief: Determine the common "availability" of API, which depends on
> the
> + *         NTF Server state
> + * @Param: None
> + * @Return: OK - If Server state is UP
> + *          TRY_AGAIN - If Server state is Down and No Active (temporary no
> active)
> + */
> +static SaAisErrorT checkNtfServerState()
> +{
> +     SaAisErrorT rc = SA_AIS_ERR_NOT_SUPPORTED;
> +
> +     osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
> +
> +     /* Check NTF server availability */
> +     if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_UP) {
> +             rc = SA_AIS_OK;
> +     } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_DOWN) {
> +             TRACE("NTFS server is down");
> +             rc = SA_AIS_ERR_TRY_AGAIN;
> +     } else if (ntfa_cb.ntfa_ntfsv_state ==
> NTFA_NTFSV_NO_ACTIVE) {
> +             TRACE("NTFS server is unavailable");
> +             rc = SA_AIS_ERR_TRY_AGAIN;
> +     } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NONE) {
> +             TRACE("No NTF server is detected, or API is
> called during headless");
> +             rc = SA_AIS_ERR_TRY_AGAIN;
> +     } else
> +             TRACE("Not supported API call under NTF Server
> state (%u)",
> +
>       ntfa_cb.ntfa_ntfsv_state);
> +
> +     osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> +
> +     return rc;
> +}
> +
> +/*
> + * @Brief: Return ntf server state in thread safe manner
> + * @Param: None
> + * @Return: Server state
> + */
> +static ntfa_ntfsv_state_t getServerState()
> +{
> +     osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
> +
> +     ntfa_ntfsv_state_t state = ntfa_cb.ntfa_ntfsv_state;
> +
> +     osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> +     return state;
> +}
> +
>  static SaAisErrorT checkNtfValueTypeRange(SaNtfValueTypeT type)
>  {
>       return (type < SA_NTF_VALUE_UINT8 || type >
> SA_NTF_VALUE_ARRAY)? SA_AIS_ERR_INVALID_PARAM
> @@ -871,7 +921,271 @@ SaAisErrorT getFilters(const SaNtfNotifi
>  }
> 
>  /* end help functions */
> -
> +/*********************************************************
> *******************
> +  Name          : reinitializeClient
> +
> +  Description   : This routine sends initialize messages to NTF Server in
> +               order to obtain new client id
> +
> +  Arguments     : client_hdl* [IN]: client handle
> +
> +  Return Values : SA_AIS_OK if success, others are failure
> +
> +  Notes         : None
> +*********************************************************
> *********************/
> +SaAisErrorT reinitializeClient(ntfa_client_hdl_rec_t* client_hdl) {
> +     uint32_t mds_rc;
> +     SaAisErrorT rc = SA_AIS_OK;
> +     ntfsv_msg_t i_msg, *o_msg = NULL;
> +
> +     TRACE_ENTER();
> +
> +     memset(&i_msg, 0, sizeof(ntfsv_msg_t));
> +     i_msg.type = NTFSV_NTFA_API_MSG;
> +     i_msg.info.api_info.type = NTFSV_INITIALIZE_REQ;
> +     i_msg.info.api_info.param.init.version = client_hdl->version;
> +
> +     mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg,
> &o_msg, NTFS_WAIT_TIME);
> +     switch (mds_rc) {
> +     case NCSCC_RC_SUCCESS:
> +             break;
> +     case NCSCC_RC_REQ_TIMOUT:
> +             rc = SA_AIS_ERR_TRY_AGAIN;
> +             goto done;
> +     default:
> +             rc = SA_AIS_ERR_BAD_HANDLE;
> +             goto done;
> +     }
> +
> +     osafassert(o_msg != NULL);
> +     /* Check response msg: type, rc */
> +     if (o_msg->info.api_resp_info.type != NTFSV_INITIALIZE_RSP)
> {
> +             TRACE("info.api_resp_info.type:%u", o_msg-
> >info.api_resp_info.type);
> +             rc = SA_AIS_ERR_LIBRARY;
> +             goto done;
> +     }
> +     if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
> +             TRACE("info.api_resp_info.rc:%u", o_msg-
> >info.api_resp_info.rc);
> +             rc = SA_AIS_ERR_BAD_HANDLE;
> +             goto done;
> +     }
> +
> +     client_hdl->ntfs_client_id = o_msg-
> >info.api_resp_info.param.init_rsp.client_id;
> +     TRACE("Successfully recover client_id, new client_id:%u",
> +                     client_hdl->ntfs_client_id);
> +
> +done:
> +     if (o_msg)
> +             ntfa_msg_destroy(o_msg);
> +     TRACE_LEAVE();
> +     return rc;
> +}
> +/*********************************************************
> *******************
> +  Name          : recoverReader
> +
> +  Description   : This routine sends read_initialize messages to NTF Server 
> in
> +               order to reintroduce this reader to NTF Server
> +
> +  Arguments     : client_hdl* [IN]: client handle
> +               reader_hdl* [IN]: reader handle
> +
> +  Return Values : SA_AIS_OK if success, others are failure
> +
> +  Notes         : None
> +*********************************************************
> *********************/
> +SaAisErrorT recoverReader(ntfa_client_hdl_rec_t* client_hdl,
> ntfa_reader_hdl_rec_t* reader_hdl) {
> +     uint32_t mds_rc;
> +     SaAisErrorT rc = SA_AIS_OK;
> +     ntfsv_msg_t i_msg, *o_msg = NULL;
> +     ntfsv_reader_init_req_2_t *send_param;
> +
> +     TRACE_ENTER();
> +
> +     /* Make sure the filters have not been deleted */
> +     if (reader_hdl->filters.alarm_filter == NULL &&
> +             reader_hdl->filters.att_ch_filter == NULL &&
> +             reader_hdl->filters.obj_cr_del_filter == NULL
> &&
> +             reader_hdl->filters.sec_al_filter == NULL &&
> +             reader_hdl->filters.sta_ch_filter == NULL)
> +             return SA_AIS_ERR_BAD_HANDLE;
> +
> +     memset(&i_msg, 0, sizeof(ntfsv_msg_t));
> +     i_msg.type = NTFSV_NTFA_API_MSG;
> +     i_msg.info.api_info.type = NTFSV_READER_INITIALIZE_REQ_2;
> +     send_param = &i_msg.info.api_info.param.reader_init_2;
> +     send_param->head.client_id = client_hdl->ntfs_client_id;
> +     send_param->head.searchCriteria = reader_hdl-
> >searchCriteria;
> +     send_param->f_rec = reader_hdl->filters;
> +
> +     mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg,
> &o_msg, NTFS_WAIT_TIME);
> +
> +     switch (mds_rc) {
> +     case NCSCC_RC_SUCCESS:
> +             break;
> +     case NCSCC_RC_REQ_TIMOUT:
> +             rc = SA_AIS_ERR_TRY_AGAIN;
> +             goto done;
> +     default:
> +             rc = SA_AIS_ERR_BAD_HANDLE;
> +             goto done;
> +     }
> +
> +     osafassert(o_msg != NULL);
> +     if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
> +             TRACE("o_msg->info.api_resp_info.rc:%u",
> o_msg->info.api_resp_info.rc);
> +             rc = SA_AIS_ERR_BAD_HANDLE;
> +             goto done;
> +     }
> +
> +     if (o_msg->info.api_resp_info.type !=
> NTFSV_READER_INITIALIZE_RSP) {
> +             TRACE("msg type (%d) failed", (int)o_msg-
> >info.api_resp_info.type);
> +             rc = SA_AIS_ERR_LIBRARY;
> +             goto done;
> +     }
> +     /* Update reader_id since it may be changed */
> +     reader_hdl->reader_id = o_msg-
> >info.api_resp_info.param.reader_init_rsp.readerId;
> +
> +     TRACE("Recover reader successfully");
> +done:
> +     if (o_msg)
> +             ntfa_msg_destroy(o_msg);
> +     TRACE_LEAVE();
> +     return rc;
> +}
> +/*********************************************************
> *******************
> +  Name          : recoverSubscriber
> +
> +  Description   : This routine sends subscribe messages to NTF Server in
> +               order to reintroduce this subscriber to NTF
> Server
> +
> +  Arguments     : client_hdl* [IN]: client handle
> +               subscriber_hdl* [IN]: subscriber handle
> +
> +  Return Values : SA_AIS_OK if success, others are failure
> +
> +  Notes         : None
> +*********************************************************
> *********************/
> +SaAisErrorT recoverSubscriber(ntfa_client_hdl_rec_t* client_hdl,
> +
> 
>       ntfa_subscriber_list_t* subscriber_hdl) {
> +     uint32_t mds_rc;
> +     SaAisErrorT rc = SA_AIS_OK;
> +     ntfsv_msg_t i_msg, *o_msg = NULL;
> +     ntfsv_subscribe_req_t *send_param;
> +
> +     TRACE_ENTER();
> +
> +     /* Make sure the filters have not been deleted */
> +     if (subscriber_hdl->filters.alarm_filter == NULL &&
> +             subscriber_hdl->filters.att_ch_filter == NULL &&
> +             subscriber_hdl->filters.obj_cr_del_filter ==
> NULL &&
> +             subscriber_hdl->filters.sec_al_filter == NULL &&
> +             subscriber_hdl->filters.sta_ch_filter == NULL)
> +             return SA_AIS_ERR_BAD_HANDLE;
> +
> +     memset(&i_msg, 0, sizeof(ntfsv_msg_t));
> +     i_msg.type = NTFSV_NTFA_API_MSG;
> +     i_msg.info.api_info.type = NTFSV_SUBSCRIBE_REQ;
> +     send_param = &i_msg.info.api_info.param.subscribe;
> +
> +     send_param->client_id = client_hdl->ntfs_client_id;
> +     send_param->subscriptionId = subscriber_hdl-
> >subscriberListSubscriptionId;
> +     send_param->f_rec = subscriber_hdl->filters;
> +
> +     mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg,
> &o_msg, NTFS_WAIT_TIME);
> +
> +     switch (mds_rc) {
> +     case NCSCC_RC_SUCCESS:
> +             break;
> +     case NCSCC_RC_REQ_TIMOUT:
> +             rc = SA_AIS_ERR_TRY_AGAIN;
> +             goto done;
> +     default:
> +             rc = SA_AIS_ERR_BAD_HANDLE;
> +             goto done;
> +     }
> +
> +     osafassert(o_msg != NULL);
> +
> +     if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
> +             TRACE("o_msg->info.api_resp_info.rc:%u",
> o_msg->info.api_resp_info.rc);
> +             rc = SA_AIS_ERR_BAD_HANDLE;
> +             goto done;
> +     }
> +
> +     if (o_msg->info.api_resp_info.type !=
> NTFSV_SUBSCRIBE_RSP) {
> +             TRACE("msg type (%d) failed", (int)o_msg-
> >info.api_resp_info.type);
> +             rc = SA_AIS_ERR_LIBRARY;
> +             goto done;
> +     }
> +     TRACE("Recover subscriber successfully");
> +done:
> +     if (o_msg)
> +             ntfa_msg_destroy(o_msg);
> +
> +     TRACE_LEAVE();
> +     return rc;
> +}
> +/*********************************************************
> *******************
> +  Name          : recoverClient
> +
> +  Description   : In NTF-A.03.01, section 2.1.2. A client can be producer,
> +               subscriber, and reader at the same time.
> +               This routine will recovery:
> +               (1) client_id
> +               (2) Then it will continue the recovery if this
> client also
> +               has instance of subscriber or reader.
> +  Arguments     : client_hdl* [IN]: client handle
> +
> +  Return Values : SA_AIS_OK if success, others are failure
> +
> +  Notes         : None
> +*********************************************************
> *********************/
> +SaAisErrorT recoverClient(ntfa_client_hdl_rec_t *client_hdl) {
> +
> +     TRACE_ENTER();
> +     SaAisErrorT rc = SA_AIS_OK;
> +
> +     osafassert(client_hdl);
> +     osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
> +
> +     if (client_hdl->valid == true)
> +             goto done;
> +
> +     if ((rc = reinitializeClient(client_hdl)) == SA_AIS_OK) {
> +             /* Restore reader */
> +             ntfa_reader_hdl_rec_t* reader_hdl =
> client_hdl->reader_list;
> +             while (reader_hdl != NULL && rc == SA_AIS_OK)
> {
> +                     rc = recoverReader(client_hdl,
> reader_hdl);
> +                     reader_hdl = reader_hdl->next;
> +             }
> +             if (rc != SA_AIS_OK) {
> +                     TRACE("Failed to restore reader
> (readerId:%d)",
> +
>       reader_hdl->reader_id);
> +                     goto done;
> +             }
> +             /* Restore subscriber */
> +             ntfa_subscriber_list_t* subscriber_hdl =
> subscriberNoList;
> +             while (subscriber_hdl != NULL && rc ==
> SA_AIS_OK) {
> +                     if (client_hdl->local_hdl ==
> subscriber_hdl->subscriberListNtfHandle)
> +                             rc =
> recoverSubscriber(client_hdl, subscriber_hdl);
> +                     subscriber_hdl = subscriber_hdl-
> >next;
> +             }
> +             if (rc != SA_AIS_OK) {
> +                     TRACE("Failed to restore
> subscriber (subscriptionId:%d)",
> +
>       subscriber_hdl->subscriberListSubscriptionId);
> +                     goto done;
> +             }
> +             client_hdl->valid = true;
> +     } else {
> +             TRACE("Failed to restore client (id:%d)",
> client_hdl->ntfs_client_id);
> +             goto done;
> +     }
> +
> +done:
> +     osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> +     TRACE_LEAVE();
> +     return rc;
> +}
> 
> /**********************************************************
> *****************
>   * 8.4.1
>   *
> @@ -923,15 +1237,14 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>               version->releaseCode = NTF_RELEASE_CODE;
>               version->majorVersion =
> NTF_MAJOR_VERSION;
>               version->minorVersion =
> NTF_MINOR_VERSION;
> -             ntfa_shutdown();
> +             ntfa_shutdown(false);
>               rc = SA_AIS_ERR_VERSION;
>               goto done;
>       }
> 
> -     if (!ntfa_cb.ntfs_up) {
> -             ntfa_shutdown();
> -             TRACE("NTFS server is down");
> -             rc = SA_AIS_ERR_TRY_AGAIN;
> +     /* Check NTF server availability */
> +     if ((rc = checkNtfServerState()) != SA_AIS_OK) {
> +             ntfa_shutdown(false);
>               goto done;
>       }
> 
> @@ -977,7 +1290,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>               rc = SA_AIS_ERR_NO_MEMORY;
>               goto err;
>       }
> -
> +     ntfa_hdl_rec->version = *version;
>       /* pass the handle value to the appl */
>       if (SA_AIS_OK == rc)
>               *ntfHandle = ntfa_hdl_rec->local_hdl;
> @@ -990,7 +1303,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
> 
>       if (rc != SA_AIS_OK) {
>               TRACE_2("NTFA INIT FAILED\n");
> -             ntfa_shutdown();
> +             ntfa_shutdown(false);
>       }
> 
>   done:
> @@ -1101,6 +1414,28 @@ SaAisErrorT saNtfDispatch(SaNtfHandleT n
>               goto done;
>       }
> 
> +     /* Need to check NTF server availability here
> +      * Because if the notificationCallback just comes after
> MDS_DOWN
> +      * it will go to recovery then result in failure due to no director
> +      */
> +     if ((rc = checkNtfServerState()) != SA_AIS_OK) {
> +             ncshm_give_hdl(ntfHandle);
> +             goto done;
> +     }
> +
> +     if (!hdl_rec->valid) {
> +             /* recovery */
> +             if ((rc = recoverClient(hdl_rec)) != SA_AIS_OK) {
> +                     if (rc ==
> SA_AIS_ERR_BAD_HANDLE) {
> +
>       ncshm_give_hdl(ntfHandle);
> +
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
> +
>       ntfa_hdl_rec_force_del(&ntfa_cb.client_list, hdl_rec);
> +
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> +
>       ntfa_shutdown(false);
> +                             goto done;
> +                     }
> +             }
> +     }
>       if ((rc = ntfa_hdl_cbk_dispatch(&ntfa_cb, hdl_rec,
> dispatchFlags)) != SA_AIS_OK)
>               TRACE("NTFA_DISPATCH_FAILURE");
> 
> @@ -1145,6 +1480,13 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
> 
>       TRACE_ENTER();
> 
> +     /* Check NTF server availability */
> +     if (getServerState() == NTFA_NTFSV_NO_ACTIVE) {
> +             TRACE("NTFS server is temporarily unavailable");
> +             rc = SA_AIS_ERR_TRY_AGAIN;
> +             goto done;
> +     }
> +
>       /* retrieve hdl rec */
>       hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
> ntfHandle);
>       if (hdl_rec == NULL) {
> @@ -1152,43 +1494,37 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>               rc = SA_AIS_ERR_BAD_HANDLE;
>               goto done;
>       }
> -
> -     /* Check Whether NTFS is up or not */
> -     if (!ntfa_cb.ntfs_up) {
> -             TRACE("NTFS down");
> -             rc = SA_AIS_ERR_TRY_AGAIN;
> -             goto done_give_hdl;
> +     if (hdl_rec->valid) {
> +             /** populate & send the finalize message
> +              ** and make sure the finalize from the server
> +              ** end returned before deleting the local
> records.
> +              **/
> +             memset(&msg, 0, sizeof(ntfsv_msg_t));
> +             msg.type = NTFSV_NTFA_API_MSG;
> +             msg.info.api_info.type = NTFSV_FINALIZE_REQ;
> +             msg.info.api_info.param.finalize.client_id =
> hdl_rec->ntfs_client_id;
> +
> +             mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb,
> &msg, &o_msg, NTFS_WAIT_TIME);
> +             switch (mds_rc) {
> +             case NCSCC_RC_SUCCESS:
> +                     break;
> +             case NCSCC_RC_REQ_TIMOUT:
> +                     rc = SA_AIS_ERR_TIMEOUT;
> +
>       TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
> +                     goto done_give_hdl;
> +             default:
> +
>       TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
> +                     rc =
> SA_AIS_ERR_NO_RESOURCES;
> +                     goto done_give_hdl;
> +             }
> +
> +             if (o_msg != NULL) {
> +                     rc = o_msg-
> >info.api_resp_info.rc;
> +                     ntfa_msg_destroy(o_msg);
> +             } else
> +                     rc =
> SA_AIS_ERR_NO_RESOURCES;
>       }
> 
> -    /** populate & send the finalize message
> -     ** and make sure the finalize from the server
> -     ** end returned before deleting the local records.
> -     **/
> -     memset(&msg, 0, sizeof(ntfsv_msg_t));
> -     msg.type = NTFSV_NTFA_API_MSG;
> -     msg.info.api_info.type = NTFSV_FINALIZE_REQ;
> -     msg.info.api_info.param.finalize.client_id = hdl_rec-
> >ntfs_client_id;
> -
> -     mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg,
> &o_msg, NTFS_WAIT_TIME);
> -     switch (mds_rc) {
> -     case NCSCC_RC_SUCCESS:
> -             break;
> -     case NCSCC_RC_REQ_TIMOUT:
> -             rc = SA_AIS_ERR_TIMEOUT;
> -             TRACE("ntfa_mds_msg_sync_send FAILED: %u",
> rc);
> -             goto done_give_hdl;
> -     default:
> -             TRACE("ntfa_mds_msg_sync_send FAILED: %u",
> rc);
> -             rc = SA_AIS_ERR_NO_RESOURCES;
> -             goto done_give_hdl;
> -     }
> -
> -     if (o_msg != NULL) {
> -             rc = o_msg->info.api_resp_info.rc;
> -             ntfa_msg_destroy(o_msg);
> -     } else
> -             rc = SA_AIS_ERR_NO_RESOURCES;
> -
>       if (rc == SA_AIS_OK) {
>       /** delete the hdl rec
>           ** including all resources allocated by client if MDS send is
> @@ -1205,7 +1541,7 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>       ncshm_give_hdl(ntfHandle);
> 
>       if (rc == SA_AIS_OK) {
> -             rc = ntfa_shutdown();
> +             rc = ntfa_shutdown(false);
>               if (rc != NCSCC_RC_SUCCESS)
>                       TRACE_1("ntfa_shutdown
> failed");
>       }
> @@ -1349,7 +1685,7 @@ SaAisErrorT saNtfNotificationFree(SaNtfN
>       }
> 
>       /* free the resources allocated by
> saNtf<ntfType>NotificationAllocate */
> -     ntfa_hdl_rec_destructor(notification_hdl_rec);
> +     ntfa_notification_destructor(notification_hdl_rec);
> 
>      /** Delete the resources related to the notificationHandle &
>       *  remove reference in the client.
> @@ -1419,6 +1755,24 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>       }
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> 
> +     /* Check NTF server availability */
> +     if ((rc = checkNtfServerState()) != SA_AIS_OK)
> +             goto done_give_hdls;
> +
> +     /* Recover if this is invalid handle */
> +     if (!client_rec->valid) {
> +             if ((rc = recoverClient(client_rec)) !=
> SA_AIS_OK) {
> +                     ncshm_give_hdl(client_handle);
> +
>       ncshm_give_hdl(notificationHandle);
> +                     if (rc ==
> SA_AIS_ERR_BAD_HANDLE) {
> +
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
> +
>       ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_rec);
> +
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> +
>       ntfa_shutdown(false);
> +                     }
> +                     goto err_free;
> +             }
> +     }
>      /**
>       ** Populate a sync MDS message
>       **/
> @@ -1481,13 +1835,6 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>               goto done_give_hdls;
>       }
> 
> -     /* Check whether NTFS is up or not */
> -     if (!ntfa_cb.ntfs_up) {
> -             TRACE("NTFS down");
> -             rc = SA_AIS_ERR_TRY_AGAIN;
> -             goto done_give_hdls;
> -     }
> -
>       /* Send a sync MDS message to obtain a notification id */
>       mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg,
> &o_msg, timeout);
>       switch (mds_rc) {
> @@ -1596,7 +1943,8 @@ static  SaNtfHandleT ntfHandleGet(SaNtfS
>       return ntfHandle;
>  }
> 
> -static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
> SaNtfSubscriptionIdT subscriptionId)
> +static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
> SaNtfSubscriptionIdT subscriptionId,
> +
> 
>       ntfsv_filter_ptrs_t filters)
>  {
>       SaAisErrorT rc = SA_AIS_OK;
>       ntfa_subscriber_list_t* ntfSubscriberList;
> @@ -1610,7 +1958,9 @@ static SaAisErrorT subscriptionListAdd(S
>       /* Add ntfHandle and subscriptionId into list */
>       ntfSubscriberList->subscriberListNtfHandle = ntfHandle;
>       ntfSubscriberList->subscriberListSubscriptionId =
> subscriptionId;
> -
> +     memset(&ntfSubscriberList->filters, 0,
> sizeof(ntfsv_filter_ptrs_t));
> +     ntfa_copy_ntf_filter_ptrs(&ntfSubscriberList->filters, &filters);
> +
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>       if (NULL == subscriberNoList) {
>               subscriberNoList = ntfSubscriberList;
> @@ -1648,6 +1998,7 @@ static void subscriberListItemRemove(SaN
>                       subscriberNoList = NULL;
>       }
>       TRACE_1("REMOVE: listPtr->SubscriptionId %d", listPtr-
> >subscriberListSubscriptionId);
> +     ntfa_del_ntf_filter_ptrs(&listPtr->filters);
>       free(listPtr);
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>  }
> @@ -1666,20 +2017,33 @@ SaAisErrorT saNtfNotificationSubscribe(c
>       ntfsv_msg_t msg, *o_msg = NULL;
>       ntfsv_subscribe_req_t *send_param;
>       uint32_t timeout = NTFS_WAIT_TIME;
> -
> +     bool recovery_failed = false;
>       TRACE_ENTER();
> +
>       rc = getFilters(notificationFilterHandles, &filters, &ntfHandle,
> &client_hdl_rec);
>       if (rc != SA_AIS_OK) {
>               TRACE("getFilters failed");
>               goto done;
>       }
> 
> +     /* Check NTF server availability */
> +     if ((rc = checkNtfServerState()) != SA_AIS_OK)
> +             goto done;
> +
> +     /* recovery */
> +     if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
> +             if ((rc = recoverClient(client_hdl_rec)) !=
> SA_AIS_OK) {
> +                     recovery_failed = true;
> +                     goto done;
> +             }
> +     }
> +
>       tmpHandle = ntfHandleGet(subscriptionId);
>       if (tmpHandle != 0) {
>               rc = SA_AIS_ERR_EXIST;
>               goto done;
>       }
> -     rc = subscriptionListAdd(ntfHandle, subscriptionId);
> +     rc = subscriptionListAdd(ntfHandle, subscriptionId, filters);
>       if (rc != SA_AIS_OK) {
>               goto done;
>       }
> @@ -1693,29 +2057,24 @@ SaAisErrorT saNtfNotificationSubscribe(c
>       send_param->client_id = client_hdl_rec->ntfs_client_id;
>       send_param->subscriptionId = subscriptionId;
>       send_param->f_rec = filters;
> -     /* Check whether NTFS is up or not */
> -     if (ntfa_cb.ntfs_up) {
> -             uint32_t rv;
> -             /* Send a sync MDS message to obtain a log
> stream id */
> -             rv = ntfa_mds_msg_sync_send(&ntfa_cb,
> &msg, &o_msg, timeout);
> -             if (rv == NCSCC_RC_SUCCESS) {
> -                     osafassert(o_msg != NULL);
> -                     if (SA_AIS_OK == o_msg-
> >info.api_resp_info.rc) {
> -
>       TRACE_1("subscriptionId from server %u",
> -
>       o_msg-
> >info.api_resp_info.param.subscribe_rsp.subscriptionId);
> -                     } else {
> -                             rc = o_msg-
> >info.api_resp_info.rc;
> -                             TRACE("Bad return
> status!!! rc = %d", rc);
> -                     }
> +
> +     uint32_t rv;
> +     /* Send a sync MDS message to obtain a log stream id */
> +     rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
> timeout);
> +     if (rv == NCSCC_RC_SUCCESS) {
> +             osafassert(o_msg != NULL);
> +             if (SA_AIS_OK == o_msg->info.api_resp_info.rc)
> {
> +                     TRACE_1("subscriptionId from
> server %u",
> +                             o_msg-
> >info.api_resp_info.param.subscribe_rsp.subscriptionId);
>               } else {
> -                     if(rv ==
> NCSCC_RC_INVALID_INPUT)
> -                             rc =
> SA_AIS_ERR_INVALID_PARAM;
> -                     else
> -                             rc =
> SA_AIS_ERR_TRY_AGAIN;
> +                     rc = o_msg-
> >info.api_resp_info.rc;
> +                     TRACE("Bad return status!!! rc =
> %d", rc);
>               }
>       } else {
> -             TRACE_1("NTFS down");
> -             rc = SA_AIS_ERR_TRY_AGAIN;
> +             if(rv == NCSCC_RC_INVALID_INPUT)
> +                     rc =
> SA_AIS_ERR_INVALID_PARAM;
> +             else
> +                     rc = SA_AIS_ERR_TRY_AGAIN;
>       }
> 
>       if (rc != SA_AIS_OK) {
> @@ -1724,6 +2083,7 @@ SaAisErrorT saNtfNotificationSubscribe(c
>       if (o_msg)
>               ntfa_msg_destroy(o_msg);
>       done:
> +
>       ncshm_give_hdl(ntfHandle);
>       if (notificationFilterHandles) {
>               if (notificationFilterHandles-
> >attributeChangeFilterHandle)
> @@ -1737,6 +2097,13 @@ SaAisErrorT saNtfNotificationSubscribe(c
>               if (notificationFilterHandles->alarmFilterHandle)
> 
>       ncshm_give_hdl(notificationFilterHandles-
> >alarmFilterHandle);
>       }
> +     if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
> +
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
> +             ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
> client_hdl_rec);
> +
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> +             ntfa_shutdown(false);
> +     }
> +
>       TRACE_LEAVE();
>       return rc;
>  }
> @@ -1995,6 +2362,7 @@ SaAisErrorT saNtfSecurityAlarmNotificati
>               rc = SA_AIS_ERR_INVALID_PARAM;
>               goto done;
>       }
> +
>       /* retrieve hdl rec */
>       hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
> ntfHandle);
>       if (hdl_rec == NULL) {
> @@ -2058,6 +2426,7 @@ SaAisErrorT saNtfPtrValAllocate(SaNtfNot
>               rc = SA_AIS_ERR_INVALID_PARAM;
>               goto done;
>       }
> +
>       notification_hdl_rec =
> ncshm_take_hdl(NCS_SERVICE_ID_NTFA, notificationHandle);
>       if (notification_hdl_rec == NULL) {
>               TRACE("ncshm_take_hdl notificationHandle
> failed");
> @@ -2098,6 +2467,7 @@ SaAisErrorT saNtfArrayValAllocate(SaNtfN
>               rc = SA_AIS_ERR_INVALID_PARAM;
>               goto done;
>       }
> +
>       notification_hdl_rec =
> ncshm_take_hdl(NCS_SERVICE_ID_NTFA, notificationHandle);
>       if (notification_hdl_rec == NULL) {
>               TRACE("ncshm_take_hdl notificationHandle
> failed");
> @@ -2228,6 +2598,7 @@ SaAisErrorT saNtfArrayValGet(SaNtfNotifi
>               rc = SA_AIS_ERR_INVALID_PARAM;
>               goto done;
>       }
> +
>       notification_hdl_rec =
> ncshm_take_hdl(NCS_SERVICE_ID_NTFA, notificationHandle);
>       if (notification_hdl_rec == NULL) {
>               TRACE("ncshm_take_hdl notificationHandle
> failed");
> @@ -2695,7 +3066,7 @@ SaAisErrorT saNtfNotificationFilterFree(
>       }
> 
>       /* free the resources allocated by
> saNtf<ntfType>FilterAllocate */
> -     ntfa_filter_hdl_rec_destructor(filter_hdl_rec);
> +     ntfa_filter_destructor(filter_hdl_rec);
> 
>      /** Delete the resources related to the notificationFilterHandle &
>       *  remove reference in the client.
> @@ -2727,9 +3098,8 @@ SaAisErrorT saNtfNotificationFilterFree(
>  SaAisErrorT saNtfNotificationUnsubscribe(SaNtfSubscriptionIdT
> subscriptionId)
>  {
>       TRACE_ENTER();
> -     SaAisErrorT rc = SA_AIS_ERR_NOT_EXIST;
> +     SaAisErrorT rc = SA_AIS_OK;
>       SaNtfHandleT ntfHandle;
> -
>       ntfa_client_hdl_rec_t *client_hdl_rec;
> 
>       ntfsv_msg_t msg, *o_msg = NULL;
> @@ -2737,6 +3107,12 @@ SaAisErrorT saNtfNotificationUnsubscribe
>       ntfsv_unsubscribe_req_t *send_param;
>       uint32_t timeout = NTFS_WAIT_TIME;
> 
> +     /* Check NTF server availability */
> +     if (getServerState() == NTFA_NTFSV_NO_ACTIVE) {
> +             TRACE("NTFS server is temporarily unavailable");
> +             rc = SA_AIS_ERR_TRY_AGAIN;
> +             goto done;
> +     }
> 
>       ntfHandle = ntfHandleGet(subscriptionId);
>       if (ntfHandle == 0) {
> @@ -2753,9 +3129,10 @@ SaAisErrorT saNtfNotificationUnsubscribe
>               goto done;
>       }
> 
> -     /**
> -         ** Populate a sync MDS message
> -         **/
> +     if (client_hdl_rec->valid) {
> +             /**
> +              ** Populate a sync MDS message
> +              **/
>               memset(&msg, 0, sizeof(ntfsv_msg_t));
>               msg.type = NTFSV_NTFA_API_MSG;
>               msg.info.api_info.type =
> NTFSV_UNSUBSCRIBE_REQ;
> @@ -2764,13 +3141,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>               send_param->client_id = client_hdl_rec-
> >ntfs_client_id;
>               send_param->subscriptionId = subscriptionId;
> 
> -             /* Check whether NTFS is up or not */
> -             if (!ntfa_cb.ntfs_up) {
> -                     TRACE_1("NTFS down");
> -                     rc = SA_AIS_ERR_TRY_AGAIN;
> -                     goto done_give_hdl;
> -             }
> -
>               /* Send a sync MDS message to obtain a log
> stream id */
>               rc = ntfa_mds_msg_sync_send(&ntfa_cb,
> &msg, &o_msg, timeout);
>               if (rc != NCSCC_RC_SUCCESS) {
> @@ -2785,6 +3155,7 @@ SaAisErrorT saNtfNotificationUnsubscribe
>                       TRACE_1("Bad return status! rc =
> %d", rc);
>                       goto done_give_hdl;
>               }
> +     }
>               subscriberListItemRemove(subscriptionId);
> 
>               /*Remove msg for subscriptionId from
> mailbox*/
> @@ -2815,6 +3186,19 @@ SaAisErrorT saNtfNotificationUnsubscribe
>   done_give_hdl:
>       if (o_msg)
>                ntfa_msg_destroy(o_msg);
> +
> +     if (!client_hdl_rec->valid && getServerState() ==
> NTFA_NTFSV_UP) {
> +             if ((rc = recoverClient(client_hdl_rec)) !=
> SA_AIS_OK) {
> +                     if (rc ==
> SA_AIS_ERR_BAD_HANDLE) {
> +
>       ncshm_give_hdl(ntfHandle);
> +
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
> +
>       ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
> +
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> +
>       ntfa_shutdown(false);
> +                             goto done;
> +                     }
> +             }
> +     }
>       ncshm_give_hdl(ntfHandle);
>   done:
>       TRACE_LEAVE();
> @@ -2838,6 +3222,7 @@ SaAisErrorT saNtfNotificationReadInitial
>       ntfsv_reader_init_req_2_t *send_param;
>       uint32_t timeout = NTFS_WAIT_TIME;
> 
> +     bool recovery_failed = false;
>       TRACE_ENTER();
>       if (notificationFilterHandles == NULL || readHandle == NULL) {
>               rc = SA_AIS_ERR_INVALID_PARAM;
> @@ -2872,6 +3257,18 @@ SaAisErrorT saNtfNotificationReadInitial
>               goto done_give_client_hdl;
>       }
> 
> +     /* Check NTF server availability */
> +     if ((rc = checkNtfServerState()) != SA_AIS_OK)
> +             goto done_give_client_hdl;
> +
> +     /* recovery */
> +     if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
> +             if ((rc = recoverClient(client_hdl_rec)) !=
> SA_AIS_OK) {
> +                     recovery_failed = true;
> +                     goto done_give_client_hdl;
> +             }
> +     }
> +
>      /**
>       ** Populate a sync MDS message
>       **/
> @@ -2884,13 +3281,6 @@ SaAisErrorT saNtfNotificationReadInitial
>       send_param->head.searchCriteria = searchCriteria;
>       send_param->f_rec = filters;
> 
> -     /* Check whether NTFS is up or not */
> -     if (!ntfa_cb.ntfs_up) {
> -             TRACE("NTFS down");
> -             rc = SA_AIS_ERR_TRY_AGAIN;
> -             goto done_give_client_hdl;
> -     }
> -
>       /* Send a sync MDS message to obtain a log stream id */
>       rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
> timeout);
>       if (rc != NCSCC_RC_SUCCESS) {
> @@ -2925,12 +3315,16 @@ SaAisErrorT saNtfNotificationReadInitial
>       reader_hdl_rec->ntfHandle = ntfHandle;
>       /* Store the readerId returned from server */
>       reader_hdl_rec->reader_id = o_msg-
> >info.api_resp_info.param.reader_init_rsp.readerId;
> +     memset(&reader_hdl_rec->filters, 0,
> sizeof(ntfsv_filter_ptrs_t));
> +     ntfa_copy_ntf_filter_ptrs(&reader_hdl_rec->filters, &filters);
> +     reader_hdl_rec->searchCriteria = searchCriteria;
>      /**                  UnLock ntfa_CB            **/
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> 
>  done_give_client_hdl:
>       if (o_msg)
>               ntfa_msg_destroy(o_msg);
> +
>     if (client_hdl_rec)
>               ncshm_give_hdl(client_hdl_rec->local_hdl);
>       if (notificationFilterHandles) {
> @@ -2947,6 +3341,12 @@ done_give_client_hdl:
>       }
> 
>       ncshm_give_hdl(notificationFilterHandles-
> >alarmFilterHandle);
> +     if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
> +
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
> +             ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
> client_hdl_rec);
> +
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> +             ntfa_shutdown(false);
> +     }
>  done:
>       TRACE_LEAVE();
>       return rc;
> @@ -2955,7 +3355,7 @@ done:
>  /*  3.15.4.2 saNtfNotificationReadFinalize()  */
>  SaAisErrorT saNtfNotificationReadFinalize(SaNtfReadHandleT readhandle)
>  {
> -     SaAisErrorT rc = SA_AIS_ERR_NOT_SUPPORTED;
> +     SaAisErrorT rc = SA_AIS_OK;
>       uint32_t oas_rc = NCSCC_RC_FAILURE;
> 
>       ntfa_client_hdl_rec_t *client_hdl_rec;
> @@ -2966,6 +3366,12 @@ SaAisErrorT saNtfNotificationReadFinaliz
>       uint32_t timeout = NTFS_WAIT_TIME;
> 
>       TRACE_ENTER();
> +     /* Check NTF server availability */
> +     if (getServerState() == NTFA_NTFSV_NO_ACTIVE) {
> +             TRACE("NTFS server is temporarily unavailable");
> +             rc = SA_AIS_ERR_TRY_AGAIN;
> +             goto done;
> +     }
> 
>       /* retrieve notification filter hdl rec */
>       reader_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
> readhandle);
> @@ -2983,42 +3389,39 @@ SaAisErrorT saNtfNotificationReadFinaliz
>       }
>       TRACE_1("reader_hdl_rec = %u", reader_hdl_rec-
> >reader_hdl);
> 
> -    /**
> -     ** Populate a sync MDS message
> -     **/
> -     memset(&msg, 0, sizeof(ntfsv_msg_t));
> -     msg.type = NTFSV_NTFA_API_MSG;
> -     msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
> -     send_param = &msg.info.api_info.param.reader_finalize;
> -     send_param->client_id = client_hdl_rec->ntfs_client_id;
> -     send_param->readerId = reader_hdl_rec->reader_id;
> -
> -     /* Check whether NTFS is up or not */
> -     if (!ntfa_cb.ntfs_up) {
> -             TRACE("NTFS down");
> -             rc = SA_AIS_ERR_TRY_AGAIN;
> -             goto done_give_hdls;
> +     if (client_hdl_rec->valid) {
> +             /**
> +              ** Populate a sync MDS message
> +              **/
> +             memset(&msg, 0, sizeof(ntfsv_msg_t));
> +             msg.type = NTFSV_NTFA_API_MSG;
> +             msg.info.api_info.type =
> NTFSV_READER_FINALIZE_REQ;
> +             send_param =
> &msg.info.api_info.param.reader_finalize;
> +             send_param->client_id = client_hdl_rec-
> >ntfs_client_id;
> +             send_param->readerId = reader_hdl_rec-
> >reader_id;
> +
> +
> +             /* Send a sync MDS message */
> +             rc = ntfa_mds_msg_sync_send(&ntfa_cb,
> &msg, &o_msg, timeout);
> +             if (rc != NCSCC_RC_SUCCESS) {
> +                     rc = SA_AIS_ERR_TRY_AGAIN;
> +                     goto done_give_hdls;
> +             }
> +
> +             osafassert(o_msg != NULL);
> +             if (SA_AIS_OK != o_msg->info.api_resp_info.rc)
> {
> +                     rc = o_msg-
> >info.api_resp_info.rc;
> +                     TRACE("Bad return status!!! rc =
> %d", rc);
> +                     goto done_give_hdls;
> +             }
> +
> +             if (o_msg->info.api_resp_info.type !=
> NTFSV_READER_FINALIZE_RSP) {
> +                     TRACE("msg type (%d) failed",
> (int)o_msg->info.api_resp_info.type);
> +                     rc = SA_AIS_ERR_LIBRARY;
> +                     goto done_give_hdls;
> +             }
>       }
> 
> -     /* Send a sync MDS message */
> -     rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
> timeout);
> -     if (rc != NCSCC_RC_SUCCESS) {
> -             rc = SA_AIS_ERR_TRY_AGAIN;
> -             goto done_give_hdls;
> -     }
> -
> -     osafassert(o_msg != NULL);
> -     if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
> -             rc = o_msg->info.api_resp_info.rc;
> -             TRACE("Bad return status!!! rc = %d", rc);
> -             goto done_give_hdls;
> -     }
> -
> -     if (o_msg->info.api_resp_info.type !=
> NTFSV_READER_FINALIZE_RSP) {
> -             TRACE("msg type (%d) failed", (int)o_msg-
> >info.api_resp_info.type);
> -             rc = SA_AIS_ERR_LIBRARY;
> -             goto done_give_hdls;
> -     }
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>       oas_rc = ntfa_reader_hdl_rec_del(&client_hdl_rec-
> >reader_list, reader_hdl_rec);
>       if (oas_rc != NCSCC_RC_SUCCESS) {
> @@ -3029,6 +3432,20 @@ SaAisErrorT saNtfNotificationReadFinaliz
>   done_give_hdls:
>       if (o_msg)
>                ntfa_msg_destroy(o_msg);
> +
> +     if (!client_hdl_rec->valid && getServerState() ==
> NTFA_NTFSV_UP) {
> +             if ((rc = recoverClient(client_hdl_rec)) !=
> SA_AIS_OK) {
> +                     if (rc ==
> SA_AIS_ERR_BAD_HANDLE) {
> +
>       ncshm_give_hdl(client_hdl_rec->local_hdl);
> +
>       ncshm_give_hdl(readhandle);
> +
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
> +
>       ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
> +
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> +
>       ntfa_shutdown(false);
> +                             goto done;
> +                     }
> +             }
> +     }
>       ncshm_give_hdl(client_hdl_rec->local_hdl);
>   done_give_read_hdl:
>       ncshm_give_hdl(readhandle);
> @@ -3082,6 +3499,23 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>       }
>       TRACE_1("reader_hdl_rec = %u", reader_hdl_rec-
> >reader_hdl);
> 
> +     /* Check NTF server availability */
> +     if ((rc = checkNtfServerState()) != SA_AIS_OK)
> +             goto done_give_hdls;
> +
> +     if (!client_hdl_rec->valid) {
> +             if ((rc = recoverClient(client_hdl_rec)) !=
> SA_AIS_OK) {
> +                     ncshm_give_hdl(client_hdl_rec-
> >local_hdl);
> +                     ncshm_give_hdl(readHandle);
> +                     if (rc ==
> SA_AIS_ERR_BAD_HANDLE) {
> +
>       osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
> +
>       ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
> +
>       osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
> +
>       ntfa_shutdown(false);
> +                     }
> +                     goto done;
> +             }
> +     }
>      /**
>       ** Populate a sync MDS message
>       **/
> @@ -3092,12 +3526,6 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>       send_param->client_id = client_hdl_rec->ntfs_client_id;
>       send_param->readerId = reader_hdl_rec->reader_id;
>       send_param->searchDirection = searchDirection;
> -     /* Check whether NTFS is up or not */
> -     if (!ntfa_cb.ntfs_up) {
> -             TRACE("NTFS down");
> -             rc = SA_AIS_ERR_TRY_AGAIN;
> -             goto done_give_hdls;
> -     }
> 
>       do {
>               /* Send a sync MDS message */
> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
> b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
> --- a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
> +++ b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
> @@ -369,7 +369,12 @@ static uint32_t ntfa_mds_svc_evt(struct
>                       TRACE("NTFS down");
> 
>       pthread_mutex_lock(&ntfa_cb.cb_lock);
> 
>       memset(&ntfa_cb.ntfs_mds_dest, 0, sizeof(MDS_DEST));
> -                     ntfa_cb.ntfs_up = 0;
> +
> +                     if (mds_cb_info-
> >info.svc_evt.i_change == NCSMDS_NO_ACTIVE) {
> +
>       ntfa_update_ntfsv_state(NTFA_NTFSV_NO_ACTIVE);
> +                     } else
> +
>       ntfa_update_ntfsv_state(NTFA_NTFSV_DOWN);
> +
> 
>       pthread_mutex_unlock(&ntfa_cb.cb_lock);
>               }
>               break;
> @@ -382,7 +387,12 @@ static uint32_t ntfa_mds_svc_evt(struct
>                       TRACE_2("MSG from NTFS
> NCSMDS_NEW_ACTIVE/UP");
> 
>       pthread_mutex_lock(&ntfa_cb.cb_lock);
>                       ntfa_cb.ntfs_mds_dest =
> mds_cb_info->info.svc_evt.i_dest;
> -                     ntfa_cb.ntfs_up = 1;
> +
> +                     if (mds_cb_info-
> >info.svc_evt.i_change == NCSMDS_NEW_ACTIVE) {
> +
>       ntfa_update_ntfsv_state(NTFA_NTFSV_NEW_ACTIVE);
> +                     } else
> +
>       ntfa_update_ntfsv_state(NTFA_NTFSV_UP);
> +
>                       if (ntfa_cb.ntfs_sync_awaited) {
>                               /* signal waiting
> thread */
> 
>       m_NCS_SEL_OBJ_IND(&ntfa_cb.ntfs_sync_sel);
> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_util.c
> b/osaf/libs/agents/saf/ntfa/ntfa_util.c
> --- a/osaf/libs/agents/saf/ntfa/ntfa_util.c
> +++ b/osaf/libs/agents/saf/ntfa/ntfa_util.c
> @@ -49,7 +49,7 @@ static unsigned int ntfa_create(void)
>       }
> 
>       /* Block and wait for indication from MDS meaning NTFS is up
> */
> -
>       osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_s
> ync_sel), 30000);
> +
>       osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_s
> ync_sel), 10000);
> 
>       pthread_mutex_lock(&ntfa_cb.cb_lock);
>       ntfa_cb.ntfs_sync_awaited = 0;
> @@ -108,7 +108,25 @@ static bool ntfa_clear_mbx(NCSCONTEXT ar
>       }
>       return true;
>  }
> +/*********************************************************
> *******************
> +  Name          : ntfa_notification_list_del
> 
> +  Description   : This routine free element allocated in list of notification
> +
> +  Arguments     : pointer to the list of notification records anchor.
> +
> +  Return Values : None
> +
> +  Notes         :
> +*********************************************************
> *********************/
> +static void ntfa_notification_list_del(ntfa_notification_hdl_rec_t
> **plstr_hdl)
> +{
> +     ntfa_notification_hdl_rec_t *lstr_hdl = *plstr_hdl;
> +     while (lstr_hdl != NULL) {
> +             ntfa_notification_destructor(lstr_hdl);
> +             lstr_hdl = lstr_hdl->next;
> +     }
> +}
> 
> /**********************************************************
> ******************
>    Name          : ntfa_notification_hdl_rec_list_del
> 
> @@ -130,6 +148,70 @@ static void ntfa_notification_hdl_rec_li
>               lstr_hdl = NULL;
>       }
>  }
> +/*********************************************************
> *******************
> +  Name          : ntfa_filter_list_del
> +
> +  Description   : This routine free element allocated in list of filter
> +
> +  Arguments     : pointer to the list of filters records anchor.
> +
> +  Return Values : None
> +
> +  Notes         :
> +*********************************************************
> *********************/
> +static void ntfa_filter_list_del(ntfa_filter_hdl_rec_t **plstr_hdl)
> +{
> +     ntfa_filter_hdl_rec_t *lstr_hdl = *plstr_hdl;
> +
> +     while (lstr_hdl != NULL) {
> +             ntfa_filter_destructor(lstr_hdl);
> +             lstr_hdl = lstr_hdl->next;
> +     }
> +}
> +/*********************************************************
> *******************
> +  Name          : ntfa_filter_hdl_rec_list_del
> +
> +  Description   : This routine deletes a list of allocated filters
> +
> +  Arguments     : pointer to the list of filters records anchor.
> +
> +  Return Values : None
> +
> +  Notes         :
> +*********************************************************
> *********************/
> +static void ntfa_filter_hdl_rec_list_del(ntfa_filter_hdl_rec_t **plstr_hdl)
> +{
> +     ntfa_filter_hdl_rec_t *lstr_hdl;
> +
> +     while ((lstr_hdl = *plstr_hdl) != NULL) {
> +             *plstr_hdl = lstr_hdl->next;
> +             ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA,
> lstr_hdl->filter_hdl);
> +             free(lstr_hdl);
> +             lstr_hdl = NULL;
> +     }
> +}
> +/*********************************************************
> *******************
> +  Name          : ntfa_reader_hdl_rec_list_del
> +
> +  Description   : This routine deletes a list of allocated readers
> +
> +  Arguments     : pointer to the list of readers records anchor.
> +
> +  Return Values : None
> +
> +  Notes         :
> +*********************************************************
> *********************/
> +static void ntfa_reader_hdl_rec_list_del(ntfa_reader_hdl_rec_t
> **plstr_hdl)
> +{
> +     ntfa_reader_hdl_rec_t *lstr_hdl;
> +     while ((lstr_hdl = *plstr_hdl) != NULL) {
> +             *plstr_hdl = lstr_hdl->next;
> +             ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA,
> lstr_hdl->reader_hdl);
> +             ntfa_del_ntf_filter_ptrs(&lstr_hdl->filters);
> +             free(lstr_hdl);
> +             lstr_hdl = NULL;
> +     }
> +}
> 
>  static SaAisErrorT ntfa_alloc_callback_notification(SaNtfNotificationsT
> *notification,
> 
>           ntfsv_send_not_req_t *not_cbk, ntfa_client_hdl_rec_t
> *hdl_rec)
> @@ -407,6 +489,10 @@ static SaAisErrorT ntfa_hdl_cbk_rec_prc(
>                       free(cbk_info-
> >param.discarded_cbk.discardedNotificationIdentifiers);
>               }
>               break;
> +     case NTFSV_DUMMY_CALLBACK:
> +             TRACE("Do nothing with dummy callback, just
> return OK");
> +             rc = SA_AIS_OK;
> +             break;
>       default:
>               TRACE("unsupported callback type: %d",
> cbk_info->type);
>               rc = SA_AIS_ERR_LIBRARY;
> @@ -544,20 +630,21 @@ unsigned int ntfa_startup(void)
>   *
>   * @return unsigned int
>   */
> -unsigned int ntfa_shutdown(void)
> +unsigned int ntfa_shutdown(bool forced)
>  {
>       unsigned int rc = NCSCC_RC_SUCCESS;
> 
> -     TRACE_ENTER2("ntfa_use_count: %u", ntfa_use_count);
> +     TRACE_ENTER2("ntfa_use_count: %u, forced: %u",
> ntfa_use_count, forced);
>       pthread_mutex_lock(&ntfa_lock);
> 
> -     if (ntfa_use_count > 1) {
> -             /* Users still exist, just decrement the use count
> */
> -             ntfa_use_count--;
> -     } else if (ntfa_use_count == 1) {
> +     if ((forced && (ntfa_use_count > 0)) || (ntfa_use_count == 1))
> {
>               ntfa_destroy();
>               rc = ncs_agents_shutdown();
>               ntfa_use_count = 0;
> +             ntfa_cb.ntfa_ntfsv_state =
> NTFA_NTFSV_NONE;
> +     } else if (ntfa_use_count > 1) {
> +             /* Users still exist, just decrement the use count
> */
> +             ntfa_use_count--;
>       }
> 
>       pthread_mutex_unlock(&ntfa_lock);
> @@ -633,7 +720,27 @@ ntfa_client_hdl_rec_t *ntfa_find_hdl_rec
> 
>       return NULL;
>  }
> +/*********************************************************
> *******************
> +  Name          : ntfa_subscriber_list_del
> 
> +  Description   : This routine deletes a list of allocated subscribers
> +
> +  Arguments     : pointer to the list of subscribers records anchor.
> +
> +  Return Values : None
> +
> +  Notes         :
> +*********************************************************
> *********************/
> +void ntfa_subscriber_list_del()
> +{
> +     ntfa_subscriber_list_t *listPtr = subscriberNoList;
> +     while (listPtr != NULL) {
> +             ntfa_subscriber_list_t* tmpSub = listPtr;
> +             listPtr = listPtr->next;
> +             free(tmpSub);
> +     }
> +     subscriberNoList = NULL;
> +}
> 
> /**********************************************************
> ******************
>    Name          : ntfa_hdl_list_del
> 
> @@ -651,15 +758,24 @@ void ntfa_hdl_list_del(ntfa_client_hdl_r
> 
>       while ((client_hdl = *p_client_hdl) != NULL) {
>               *p_client_hdl = client_hdl->next;
> +             m_NCS_IPC_DETACH(&client_hdl->mbx,
> ntfa_clear_mbx, NULL);
> +             m_NCS_IPC_RELEASE(&client_hdl->mbx, NULL);
>               ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA,
> client_hdl->local_hdl);
>       /** clean up the channel records for this ntfa-client
>           **/
> +             ntfa_notification_list_del(&client_hdl-
> >notification_list);
>               ntfa_notification_hdl_rec_list_del(&client_hdl-
> >notification_list);
> +
> +             ntfa_filter_list_del(&client_hdl->filter_list);
> +             ntfa_filter_hdl_rec_list_del(&client_hdl-
> >filter_list);
> +
> +             ntfa_reader_hdl_rec_list_del(&client_hdl-
> >reader_list);
>       /** remove the association with hdl-mngr
>           **/
>               free(client_hdl);
>               client_hdl = 0;
>       }
> +     ntfa_subscriber_list_del();
>  }
> 
> 
> /**********************************************************
> ******************
> @@ -757,10 +873,80 @@ uint32_t ntfa_filter_hdl_rec_del(ntfa_fi
>       TRACE("The node couldn't be deleted");
>       return NCSCC_RC_FAILURE;
>  }
> +/*********************************************************
> *******************
> +  Name          : ntfa_hdl_rec_force_del
> 
> +  Description   : This routine deletes all memory allocated to client.
> +
> +  Arguments     : NTFA_CLIENT_HDL_REC **list_head
> +               NTFA_CLIENT_HDL_REC *rm_node
> +
> +  Return Values : None
> +
> +  Notes         :
> +*********************************************************
> *********************/
> +void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **list_head,
> ntfa_client_hdl_rec_t *rm_node)
> +{
> +     ntfa_client_hdl_rec_t *list_iter = *list_head;
> +     TRACE_ENTER();
> +     /* First remove the rm_node out of the list of client */
> +     if (list_iter == rm_node)
> +             *list_head = rm_node->next;
> +     else {
> +             while (list_iter) {
> +                     if (list_iter->next == rm_node) {
> +                             list_iter->next =
> rm_node->next;
> +                             break;
> +                     }
> +                     list_iter = list_iter->next;
> +             }
> +     }
> +     /* Release all msgs in mailbox */
> +     ntfsv_msg_t *cbk_msg;
> +     while((cbk_msg =
> (ntfsv_msg_t*)m_NCS_IPC_NON_BLK_RECEIVE(&rm_node->mbx,
> cbk_msg))
> +             != NULL) {
> +             ntfa_msg_destroy(cbk_msg);
> +     }
> +     /* delete subscriber of this client out of the subcriberNoList*/
> +     ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
> +     while (subscriber_hdl != NULL) {
> +             ntfa_subscriber_list_t *rm_subscriber =
> subscriber_hdl;
> +             subscriber_hdl = subscriber_hdl->next;
> +             if (rm_node->local_hdl == rm_subscriber-
> >subscriberListNtfHandle) {
> +                     if (rm_subscriber->next != NULL) {
> +                             rm_subscriber-
> >next->prev = rm_subscriber->prev;
> +                     }
> +
> +                     if (rm_subscriber->prev != NULL) {
> +                             rm_subscriber-
> >prev->next = rm_subscriber->next;
> +                     } else {
> +                             if (rm_subscriber-
> >next != NULL)
> +
>       subscriberNoList = rm_subscriber->next;
> +                             else
> +
>       subscriberNoList = NULL;
> +                     }
> +
>       ntfa_del_ntf_filter_ptrs(&rm_subscriber->filters);
> +                     free(rm_subscriber);
> +             }
> +     }
> +     /* Now delete client */
> +     m_NCS_IPC_DETACH(&rm_node->mbx, ntfa_clear_mbx,
> NULL);
> +     m_NCS_IPC_RELEASE(&rm_node->mbx, NULL);
> +     ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node-
> >local_hdl);
> +     ntfa_notification_list_del(&rm_node->notification_list);
> +     ntfa_notification_hdl_rec_list_del(&rm_node-
> >notification_list);
> +
> +     ntfa_filter_list_del(&rm_node->filter_list);
> +     ntfa_filter_hdl_rec_list_del(&rm_node->filter_list);
> +
> +     ntfa_reader_hdl_rec_list_del(&rm_node->reader_list);
> +     free(rm_node);
> +
> +     TRACE_LEAVE();
> +}
> 
> /**********************************************************
> ******************
>    Name          : ntfa_hdl_rec_del
> -
> +
>    Description   : This routine deletes the a client handle record from
>                    a list of client hdl records.
> 
> @@ -962,7 +1148,7 @@ ntfa_client_hdl_rec_t *ntfa_hdl_rec_add(
>      /** Associate with the client_id obtained from NTFS
>       **/
>       rec->ntfs_client_id = client_id;
> -
> +     rec->valid = true;
>      /** Initialize and attach the IPC/Priority queue
>       **/
> 
> @@ -1063,7 +1249,7 @@ static void logtrace_init_constructor(vo
>   *
>   * @param instance
>   */
> -void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t *instance)
> +void ntfa_notification_destructor(ntfa_notification_hdl_rec_t *instance)
>  {
>       ntfa_notification_hdl_rec_t *notificationInstance = instance;
> 
> @@ -1111,7 +1297,7 @@ void ntfa_hdl_rec_destructor(ntfa_notifi
>   *
>   * @param instance
>   */
> -void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t *filter_rec)
> +void ntfa_filter_destructor(ntfa_filter_hdl_rec_t *filter_rec)
>  {
>       switch (filter_rec->ntfType) {
>       case SA_NTF_TYPE_OBJECT_CREATE_DELETE:
> @@ -1202,6 +1388,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>           **/
>               ncshm_give_hdl(rm_node->reader_hdl);
>               ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA,
> rm_node->reader_hdl);
> +             ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>               free(rm_node);
>               return NCSCC_RC_SUCCESS;
>       } else {                /* find the rec */
> @@ -1213,6 +1400,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>                   **/
> 
>       ncshm_give_hdl(rm_node->reader_hdl);
> 
>       ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node-
> >reader_hdl);
> +
>       ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>                               free(rm_node);
>                               return
> NCSCC_RC_SUCCESS;
>                       }
> @@ -1258,3 +1446,258 @@ void ntfa_add_to_async_cbk_msg_list(ntfs
> 
>       TRACE_LEAVE();
>  }
> +
> +/*********************************************************
> *******************
> +  Name          : ntfa_notify_handle_invalid
> +
> +  Description   : This routine sends a dummy callback msg to client's mailbox
> +               so that the client polls in and calls
> saNtfDispatch.
> +  Arguments     :
> +
> +  Return Values : None
> +
> +  Notes         : None
> +*********************************************************
> *********************/
> +void ntfa_notify_handle_invalid() {
> +     ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
> +     TRACE_ENTER();
> +     while (client_hdl != NULL) {
> +             /* Only applicable for subscriber */
> +             if (client_hdl->reg_cbk.saNtfNotificationCallback
> != NULL ||
> +                     client_hdl-
> >reg_cbk.saNtfNotificationDiscardedCallback != NULL) {
> +                     /* Create a dummy msg */
> +                     ntfsv_msg_t *msg =
> malloc(sizeof(ntfsv_msg_t));
> +                     memset(msg, 0,
> sizeof(ntfsv_msg_t));
> +                     msg->info.cbk_info.type =
> NTFSV_DUMMY_CALLBACK;
> +                     /* Send dummy msg to client
> mailbox */
> +                     if
> (m_NCS_IPC_SEND(&client_hdl->mbx, msg, MDS_SEND_PRIORITY_HIGH) !=
> NCSCC_RC_SUCCESS) {
> +
>       TRACE_1("m_NCS_IPC_SEND Failed to client(id:%u)",
> client_hdl->ntfs_client_id);
> +
>       ntfa_msg_destroy(msg);
> +                     }
> +             }
> +
> +             client_hdl = client_hdl->next;
> +     }
> +     TRACE_LEAVE();
> +}
> +/*********************************************************
> *******************
> +  Name          : ntfa_update_ntfsv_state
> +
> +  Description   : Update current NTF Server state by the @changedState
> indicated
> +               by MDS event
> +  Arguments     : @changedState [IN]: state to be changed of NTF Server
> +
> +  Return Values : None
> +
> +  Notes         : None
> +*********************************************************
> *********************/
> +void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState)
> +{
> +     TRACE_ENTER();
> +     TRACE_1("Current state: %u, Changed state: %u",
> ntfa_cb.ntfa_ntfsv_state,
> +
> 
>               changedState);
> +
> +     ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
> +
> +     switch (ntfa_cb.ntfa_ntfsv_state){
> +     case NTFA_NTFSV_NONE:
> +             ntfa_cb.ntfa_ntfsv_state = changedState;
> +             break;
> +     case NTFA_NTFSV_DOWN:
> +             if (changedState == NTFA_NTFSV_NEW_ACTIVE
> ||
> +                     changedState ==
> NTFA_NTFSV_UP) {
> +                     TRACE("Active NTF server has
> been restarted");
> +                     ntfa_cb.ntfa_ntfsv_state =
> NTFA_NTFSV_UP;
> +                     ntfa_notify_handle_invalid();
> +             } else
> +                     TRACE("Unexpected state
> changes");
> +             break;
> +     case NTFA_NTFSV_NO_ACTIVE:
> +             if (changedState ==
> NTFA_NTFSV_NEW_ACTIVE) {
> +                     TRACE("Standby NTF server
> becomes new Active");
> +                     /* NTF server is functioning
> normally */
> +                     ntfa_cb.ntfa_ntfsv_state =
> NTFA_NTFSV_UP;
> +             } else if (changedState == NTFA_NTFSV_DOWN)
> {
> +                     TRACE("Active NTF Server is
> Down");
> +                     ntfa_cb.ntfa_ntfsv_state =
> NTFA_NTFSV_DOWN;
> +                     /* Mark all client handles are
> invalid */
> +                     while (client_hdl != NULL) {
> +                             client_hdl->valid =
> false;
> +                             client_hdl =
> client_hdl->next;
> +                     }
> +             }
> +             break;
> +     case NTFA_NTFSV_NEW_ACTIVE:
> +             TRACE("Unknown");
> +             break;
> +     case NTFA_NTFSV_UP:
> +             if (changedState == NTFA_NTFSV_NO_ACTIVE) {
> +                     TRACE("Active NTF server
> temporarily unavailable");
> +                     /* Failover/Switchover is
> happening
> +                      * Any API calls result in
> TRY_AGAIN
> +                      */
> +                     ntfa_cb.ntfa_ntfsv_state =
> NTFA_NTFSV_NO_ACTIVE;
> +             } else
> +                     TRACE("Unexpected state
> changes");
> +             break;
> +     default:
> +             osafassert(false);
> +     }
> +
> +     TRACE_LEAVE();
> +}
> +/*********************************************************
> *******************
> +  Name          : ntfa_copy_ntf_filter_ptrs
> +
> +  Description   : Copy a list of filter from pSrc to pDes
> +
> +  Arguments     : pDes* [OUT]: list of outcome filter
> +               pSrc* [IN]: list of input filter
> +
> +  Return Values : SA_AIS_OK if succeed, other values as failed
> +
> +  Notes         : None
> +*********************************************************
> *********************/
> +SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
> +
>                       const ntfsv_filter_ptrs_t* pSrc) {
> +     SaAisErrorT rc = SA_AIS_OK;
> +     SaNtfNotificationFilterHeaderT *des_header;
> +     SaNtfNotificationFilterHeaderT *src_header;
> +     TRACE_ENTER();
> +
> +     if (pSrc->alarm_filter) {
> +             pDes->alarm_filter = calloc(1,
> sizeof(SaNtfAlarmNotificationFilterT));
> +             des_header = &(pDes->alarm_filter-
> >notificationFilterHeader);
> +             src_header = &(pSrc->alarm_filter-
> >notificationFilterHeader);
> +             if ((rc = ntfsv_filter_header_alloc(des_header,
> src_header->numEventTypes,
> +
> 
>       src_header->numNotificationObjects,
> +
> 
>       src_header->numNotifyingObjects,
> +
> 
>       src_header->numNotificationClassIds)) != SA_AIS_OK)
> +                     goto done;
> +             if ((rc = ntfsv_filter_alarm_alloc(pDes-
> >alarm_filter,
> +
> 
>       pSrc->alarm_filter->numProbableCauses,
> +
> 
>       pSrc->alarm_filter->numPerceivedSeverities,
> +
> 
>       pSrc->alarm_filter->numTrends)) != SA_AIS_OK)
> +                     goto done;
> +             if ((rc = ntfsv_copy_ntf_filter_alarm(pDes-
> >alarm_filter,
> +
> 
>               pSrc->alarm_filter)) != SA_AIS_OK)
> +                     goto done;
> +     }
> +
> +     if (pSrc->sec_al_filter) {
> +             pDes->sec_al_filter = calloc(1,
> sizeof(SaNtfSecurityAlarmNotificationFilterT));
> +             des_header = &(pDes->sec_al_filter-
> >notificationFilterHeader);
> +             src_header = &(pSrc->sec_al_filter-
> >notificationFilterHeader);
> +             if ((rc = ntfsv_filter_header_alloc(des_header,
> src_header->numEventTypes,
> +
> 
>       src_header->numNotificationObjects,
> +
> 
>       src_header->numNotifyingObjects,
> +
> 
>       src_header->numNotificationClassIds)) != SA_AIS_OK)
> +                     goto done;
> +             if ((rc = ntfsv_filter_sec_alarm_alloc(pDes-
> >sec_al_filter,
> +
> 
>       pSrc->sec_al_filter->numProbableCauses,
> +
> 
>       pSrc->sec_al_filter->numSeverities,
> +
> 
>       pSrc->sec_al_filter->numSecurityAlarmDetectors,
> +
> 
>       pSrc->sec_al_filter->numServiceUsers,
> +
> 
>       pSrc->sec_al_filter->numServiceProviders)) != SA_AIS_OK)
> +                     goto done;
> +             if ((rc = ntfsv_copy_ntf_filter_sec_alarm(pDes-
> >sec_al_filter,
> +
> 
>               pSrc->sec_al_filter)) != SA_AIS_OK)
> +                     goto done;
> +     }
> +
> +     if (pSrc->sta_ch_filter) {
> +             pDes->sta_ch_filter = calloc(1,
> sizeof(SaNtfStateChangeNotificationFilterT));
> +             des_header = &(pDes->sta_ch_filter-
> >notificationFilterHeader);
> +             src_header = &(pSrc->sta_ch_filter-
> >notificationFilterHeader);
> +             if ((rc = ntfsv_filter_header_alloc(des_header,
> src_header->numEventTypes,
> +
> 
>       src_header->numNotificationObjects,
> +
> 
>       src_header->numNotifyingObjects,
> +
> 
>       src_header->numNotificationClassIds)) != SA_AIS_OK)
> +                     goto done;
> +             if ((rc = ntfsv_filter_state_ch_alloc(pDes-
> >sta_ch_filter,
> +
> 
>       pSrc->sta_ch_filter->numSourceIndicators,
> +
> 
>       pSrc->sta_ch_filter->numStateChanges)) != SA_AIS_OK)
> +                     goto done;
> +             if ((rc = ntfsv_copy_ntf_filter_state_ch(pDes-
> >sta_ch_filter,
> +
> 
>               pSrc->sta_ch_filter)) != SA_AIS_OK)
> +                     goto done;
> +     }
> +
> +     if (pSrc->obj_cr_del_filter) {
> +             pDes->obj_cr_del_filter = calloc(1,
> sizeof(SaNtfObjectCreateDeleteNotificationFilterT));
> +             des_header = &(pDes->obj_cr_del_filter-
> >notificationFilterHeader);
> +             src_header = &(pSrc->obj_cr_del_filter-
> >notificationFilterHeader);
> +             if ((rc = ntfsv_filter_header_alloc(des_header,
> src_header->numEventTypes,
> +
> 
>       src_header->numNotificationObjects,
> +
> 
>       src_header->numNotifyingObjects,
> +
> 
>       src_header->numNotificationClassIds)) != SA_AIS_OK)
> +                     goto done;
> +             if ((rc = ntfsv_filter_obj_cr_del_alloc(pDes-
> >obj_cr_del_filter,
> +
> 
>       pSrc->obj_cr_del_filter->numSourceIndicators)) !=
> SA_AIS_OK)
> +                     goto done;
> +             if ((rc = ntfsv_copy_ntf_filter_obj_cr_del(pDes-
> >obj_cr_del_filter,
> +
> 
>               pSrc->obj_cr_del_filter)) != SA_AIS_OK)
> +                     goto done;
> +     }
> +
> +     if (pSrc->att_ch_filter) {
> +             pDes->att_ch_filter = calloc(1,
> sizeof(SaNtfAttributeChangeNotificationFilterT));
> +             des_header = &(pDes->att_ch_filter-
> >notificationFilterHeader);
> +             src_header = &(pSrc->att_ch_filter-
> >notificationFilterHeader);
> +             if ((rc = ntfsv_filter_header_alloc(des_header,
> src_header->numEventTypes,
> +
> 
>       src_header->numNotificationObjects,
> +
> 
>       src_header->numNotifyingObjects,
> +
> 
>       src_header->numNotificationClassIds)) != SA_AIS_OK)
> +                     goto done;
> +             if ((rc = ntfsv_filter_attr_change_alloc(pDes-
> >att_ch_filter,
> +
> 
>       pSrc->att_ch_filter->numSourceIndicators)) != SA_AIS_OK)
> +                     goto done;
> +             if ((rc = ntfsv_copy_ntf_filter_attr_ch(pDes-
> >att_ch_filter,
> +
> 
>               pSrc->att_ch_filter)) != SA_AIS_OK)
> +                     goto done;
> +     }
> +done:
> +     TRACE_LEAVE();
> +     return rc;
> +}
> +
> +/*********************************************************
> *******************
> +  Name          : ntfa_del_ntf_filter_ptrs
> +
> +  Description   : Delete the filter pointers
> +
> +  Arguments     : filter_ptrs* [IN/OUT]: filter pointers
> +
> +  Return Values : SA_AIS_OK if succeed, other values as failed
> +
> +  Notes         : None
> +*********************************************************
> *********************/
> +SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t* filter_ptrs)
> +{
> +     SaAisErrorT rc = SA_AIS_OK;
> +     if (filter_ptrs->alarm_filter) {
> +             ntfsv_filter_alarm_free(filter_ptrs-
> >alarm_filter, true);
> +             free(filter_ptrs->alarm_filter);
> +     }
> +
> +     if (filter_ptrs->sec_al_filter) {
> +             ntfsv_filter_sec_alarm_free(filter_ptrs-
> >sec_al_filter, true);
> +             free(filter_ptrs->sec_al_filter);
> +     }
> +
> +     if (filter_ptrs->sta_ch_filter) {
> +             ntfsv_filter_state_ch_free(filter_ptrs-
> >sta_ch_filter, true);
> +             free(filter_ptrs->sta_ch_filter);
> +     }
> +
> +     if (filter_ptrs->obj_cr_del_filter) {
> +             ntfsv_filter_obj_cr_del_free(filter_ptrs-
> >obj_cr_del_filter, true);
> +             free(filter_ptrs->obj_cr_del_filter);
> +     }
> +
> +     if (filter_ptrs->att_ch_filter) {
> +             ntfsv_filter_attr_ch_free(filter_ptrs-
> >att_ch_filter, true);
> +             free(filter_ptrs->att_ch_filter);
> +     }
> +     return rc;
> +}

------------------------------------------------------------------------------
Transform Data into Opportunity.
Accelerate data analysis in your applications with
Intel Data Analytics Acceleration Library.
Click to learn more.
http://pubads.g.doubleclick.net/gampad/clk?id=278785231&iu=/4140
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel

Reply via email to