Hi Vu,

Please see comments in line with [Minh]

Thanks,
Minh

On 25/02/16 17:57, Vu Minh Nguyen wrote:
> Hi Minh,
>
> I have few comments below [Vu] and one question.
>
> I see, in some places,  NTF APIs not always return TRY_AGAIN if both SCs
> down.
> I am not sure if I feel correctly or not.
>
> E.g: In `saNtfNotificationSend` API
> When the client thread comes to code line `ntfa_mds_msg_sync_send()`,
> headless occurs. If this is the case, the API may return TIMEOUT.
[Minh] I think we should return TIMEOUT here, respect the fact that mds 
call get timeout. client shall also try again with timeout (#1607)
> Regards, Vu.
>
>
>> -----Original Message-----
>> From: Minh Hon Chau [mailto:minh.c...@dektech.com.au]
>> Sent: Wednesday, December 23, 2015 11:02 AM
>> To: lennart.l...@ericsson.com; praveen.malv...@oracle.com;
>> vu.m.ngu...@dektech.com.au
>> Cc: opensaf-devel@lists.sourceforge.net
>> Subject: [PATCH 1 of 5] NTF: Add support cloud resilience for NTF Agent
> [#1180]
>> osaf/libs/agents/saf/ntfa/ntfa.h      |   31 +-
>> osaf/libs/agents/saf/ntfa/ntfa_api.c  |  672
> +++++++++++++++++++++++++++----
>> --
>> osaf/libs/agents/saf/ntfa/ntfa_mds.c  |    6 +-
>> osaf/libs/agents/saf/ntfa/ntfa_util.c |  465 ++++++++++++++++++++++-
>> 4 files changed, 1022 insertions(+), 152 deletions(-)
>>
>>
>> The patch contains support for cloud resilience feature
>> in NTF Agent code. Please refer README.HYDRA for content
>> of the changes
>>
>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa.h
> b/osaf/libs/agents/saf/ntfa/ntfa.h
>> --- a/osaf/libs/agents/saf/ntfa/ntfa.h
>> +++ b/osaf/libs/agents/saf/ntfa/ntfa.h
>> @@ -91,6 +91,7 @@ typedef struct ntfa_filter_hdl_rec {
>> typedef struct subscriberList {
>>      SaNtfHandleT subscriberListNtfHandle;
>>      SaNtfSubscriptionIdT subscriberListSubscriptionId;
>> +    ntfsv_filter_ptrs_t filters; /* remember the filters used by this
>> subscriber */
>>      struct subscriberList *prev;
>>      struct subscriberList *next;
>> } ntfa_subscriber_list_t;
>> @@ -100,6 +101,10 @@ typedef struct ntfa_reader_hdl_rec {
>>      unsigned int reader_id; /* handle value returned by NTFS for this
> client
>> */
>>      SaNtfHandleT ntfHandle;
>>      unsigned int reader_hdl;        /* READER handle from handle mgr */
>> +
>> +    ntfsv_filter_ptrs_t filters; /* remember the filters used by this
> reader
>> */
>> +    SaNtfSearchCriteriaT searchCriteria; /* remember the searchCriteria
> for
>> recovery */
>> +
>>      struct ntfa_reader_hdl_rec *next;       /* next pointer for the list
> in
>> ntfa_cb_t */
>>      struct ntfa_client_hdl_rec *parent_hdl; /* Back Pointer to the
> client
>> instantiation */
>> } ntfa_reader_hdl_rec_t;
>> @@ -114,24 +119,35 @@ typedef struct ntfa_client_hdl_rec {
>>      ntfa_reader_hdl_rec_t *reader_list;
>>      SYSF_MBX mbx;           /* priority q mbx b/w MDS & Library */
>>      struct ntfa_client_hdl_rec *next;       /* next pointer for the list
> in
>> ntfa_cb_t */
>> +    bool valid;             /* handle is valid if it's known by NTF
> server,
>> used for headless hydra */
>> +    SaVersionT version; /* the API version is being used by client, used
> for
>> recover after headless */
>> } ntfa_client_hdl_rec_t;
>>
>> /*
>>   * The NTFA control block is the master anchor structure for all NTFA
>>   * instantiations within a process.
>>   */
>> +typedef enum {
>> +    NTFA_NTFSV_NONE = 0,
>> +    NTFA_NTFSV_DOWN,
>> +    NTFA_NTFSV_NO_ACTIVE,
>> +    NTFA_NTFSV_NEW_ACTIVE,
>> +    NTFA_NTFSV_UP
>> +}ntfa_ntfsv_state_t;
>> +
>> typedef struct {
>>      pthread_mutex_t cb_lock;        /* CB lock */
>>      ntfa_client_hdl_rec_t *client_list;     /* NTFA client handle
> database
>> */
>>      ntfa_reader_hdl_rec_t *reader_list;
>>      MDS_HDL mds_hdl;        /* MDS handle */
>>      MDS_DEST ntfs_mds_dest; /* NTFS absolute/virtual address */
>> -    int ntfs_up;            /* Indicate that MDS subscription
>> -                             * is complete */
>> +
>>      /* NTFS NTFA sync params */
>>      int ntfs_sync_awaited;
>>      NCS_SEL_OBJ ntfs_sync_sel;
>>      SaUint32T ntf_var_data_limit;   /* max allowed variableDataSize */
>> +    /* NTF Server state */
>> +    ntfa_ntfsv_state_t ntfa_ntfsv_state;
>> } ntfa_cb_t;
>>
>> /* ntfa_saf_api.c */
>> @@ -149,7 +165,7 @@ extern void ntfsv_ntfa_evt_free(struct n
>>
>> /* ntfa_init.c */
>> extern unsigned int ntfa_startup(void);
>> -extern unsigned int ntfa_shutdown(void);
>> +extern unsigned int ntfa_shutdown(bool forced);
>>
>> /* ntfa_hdl.c */
>> extern SaAisErrorT ntfa_hdl_cbk_dispatch(ntfa_cb_t *,
> ntfa_client_hdl_rec_t *,
>> SaDispatchFlagsT);
>> @@ -159,6 +175,7 @@ extern ntfa_notification_hdl_rec_t *ntfa
>> extern ntfa_filter_hdl_rec_t
> *ntfa_filter_hdl_rec_add(ntfa_client_hdl_rec_t
>> **hdl_rec);
>> extern void ntfa_hdl_list_del(ntfa_client_hdl_rec_t **);
>> extern uint32_t ntfa_hdl_rec_del(ntfa_client_hdl_rec_t **,
>> ntfa_client_hdl_rec_t *);
>> +extern void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **,
>> ntfa_client_hdl_rec_t *);
>> extern uint32_t ntfa_notification_hdl_rec_del(ntfa_notification_hdl_rec_t
> **,
>> ntfa_notification_hdl_rec_t *);
>> extern uint32_t ntfa_filter_hdl_rec_del(ntfa_filter_hdl_rec_t **,
>> ntfa_filter_hdl_rec_t *);
>> extern bool ntfa_validate_ntfa_client_hdl(ntfa_cb_t *ntfa_cb,
>> ntfa_client_hdl_rec_t *find_hdl_rec);
>> @@ -166,11 +183,15 @@ extern bool ntfa_validate_ntfa_client_hd
>> /* ntfa_util.c */
>> extern ntfa_client_hdl_rec_t *ntfa_find_hdl_rec_by_client_id(ntfa_cb_t
>> *ntfa_cb, uint32_t client_id);
>> extern void ntfa_msg_destroy(ntfsv_msg_t *msg);
>> -extern void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t
> *instance);
>> -extern void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t
>> +extern void ntfa_notification_destructor(ntfa_notification_hdl_rec_t
>> *instance);
>> +extern void ntfa_filter_destructor(ntfa_filter_hdl_rec_t
>>                                         *notificationFilterInstance);
>> extern ntfa_reader_hdl_rec_t
> *ntfa_reader_hdl_rec_add(ntfa_client_hdl_rec_t
>> **hdl_rec);
>> extern uint32_t ntfa_reader_hdl_rec_del(ntfa_reader_hdl_rec_t **,
>> ntfa_reader_hdl_rec_t *);
>> extern void ntfa_add_to_async_cbk_msg_list(ntfsv_msg_t ** head,
>> ntfsv_msg_t * new_node);
>> extern uint32_t ntfa_ntfs_msg_proc(ntfa_cb_t *cb, ntfsv_msg_t *ntfsv_msg,
>> MDS_SEND_PRIORITY_TYPE prio);
>> +extern void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState);
>> +extern SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
>> +                                                            const
>> ntfsv_filter_ptrs_t* pSrc);
>> +extern SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t*
> filter_ptrs);
>> #endif   /* !NTFA_H */
>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_api.c
>> b/osaf/libs/agents/saf/ntfa/ntfa_api.c
>> --- a/osaf/libs/agents/saf/ntfa/ntfa_api.c
>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_api.c
>> @@ -34,11 +34,42 @@
>> /* The main controle block */
>> ntfa_cb_t ntfa_cb = {
>>      .cb_lock = PTHREAD_MUTEX_INITIALIZER,
>> +    .ntfa_ntfsv_state = NTFA_NTFSV_NONE,
>> };
>>
>> /* list of subscriptions for this process */
>> ntfa_subscriber_list_t *subscriberNoList = NULL;
>>
>> +/*
>> + * @Brief: Determine the common "availability" of API, which depends on
> the
>> + *         NTF Server state
>> + * @Param: None
>> + * @Return: OK - If Server state is UP
>> + *          TRY_AGAIN - If Server state is No Active (temporary no active)
>> + *          BAD_HANDLE - If Server state is completely Down (no active at
> all).
>> + *                       This error code will be returned to client
> calling API
>> + */
>> +static SaAisErrorT checkNtfServerState()
>> +{
>> +    SaAisErrorT rc = SA_AIS_ERR_NOT_SUPPORTED;
>> +    /* Check NTF server availability */
>> +    if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_UP) {
>> +            rc = SA_AIS_OK;
>> +    } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_DOWN) {
>> +            TRACE("NTFS server is down");
>> +            rc = SA_AIS_ERR_TRY_AGAIN;
>> +    } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NO_ACTIVE) {
>> +            TRACE("NTFS server is unavailable");
>> +            rc = SA_AIS_ERR_TRY_AGAIN;
>> +    } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NONE) {
>> +            TRACE("No NTF server is detected, or API is called during
>> headless");
>> +            rc = SA_AIS_ERR_TRY_AGAIN;
>> +    } else
>> +            TRACE("Not supported API call under NTF Server state (%u)",
>> +                            ntfa_cb.ntfa_ntfsv_state);
>> +    return rc;
>> +}
>> +
>> static SaAisErrorT checkNtfValueTypeRange(SaNtfValueTypeT type)
>> {
>>      return (type < SA_NTF_VALUE_UINT8 || type >
>> SA_NTF_VALUE_ARRAY)? SA_AIS_ERR_INVALID_PARAM
>> @@ -871,7 +902,266 @@ SaAisErrorT getFilters(const SaNtfNotifi
>> }
>>
>> /* end help functions */
>> -
>> +/****************************************************************
>> ************
>> +  Name          : reinitializeClient
>> +
>> +  Description   : This routine sends initialize messages to NTF Server in
>> +              order to obtain new client id
>> +
>> +  Arguments     : client_hdl* [IN]: client handle
>> +
>> +  Return Values : SA_AIS_OK if success, others are failure
>> +
>> +  Notes         : None
>> +*****************************************************************
>> *************/
>> +SaAisErrorT reinitializeClient(ntfa_client_hdl_rec_t* client_hdl) {
> [Vu] This function and other ones, recoverReader/Client/Subscriber are only
> used internally in ntfa_api.c file.
> Should define them as `static` functions?
[Minh] I don't think any reason to limit this function so we make it static
>
>> +    uint32_t mds_rc;
>> +    SaAisErrorT rc = SA_AIS_OK;
>> +    ntfsv_msg_t i_msg, *o_msg = NULL;
>> +
>> +    TRACE_ENTER();
>> +
>> +    memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>> +    i_msg.type = NTFSV_NTFA_API_MSG;
>> +    i_msg.info.api_info.type = NTFSV_INITIALIZE_REQ;
>> +    i_msg.info.api_info.param.init.version = client_hdl->version;
>> +
>> +    mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>> NTFS_WAIT_TIME);
>> +    switch (mds_rc) {
>> +    case NCSCC_RC_SUCCESS:
>> +            break;
>> +    case NCSCC_RC_REQ_TIMOUT:
>> +            rc = SA_AIS_ERR_TRY_AGAIN;
>> +            goto done;
>> +    default:
>> +            rc = SA_AIS_ERR_BAD_HANDLE;
>> +            goto done;
>> +    }
>> +
>> +    osafassert(o_msg != NULL);
>> +    /* Check response msg: type, rc */
>> +    if (o_msg->info.api_resp_info.type != NTFSV_INITIALIZE_RSP) {
>> +            TRACE("info.api_resp_info.type:%u", o_msg-
>>> info.api_resp_info.type);
>> +            rc = SA_AIS_ERR_LIBRARY;
>> +            goto done;
>> +    }
>> +    if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>> +            TRACE("info.api_resp_info.rc:%u", o_msg-
>>> info.api_resp_info.rc);
>> +            rc = SA_AIS_ERR_BAD_HANDLE;
>> +            goto done;
>> +    }
> [Vu] Is there any reason not return error code `rc`, but BAD_HANDLE here?
[Minh] Return BAD_HANDLE to "inform" client reinitialize handle since 
it's failed to recover
>
>> +
>> +    client_hdl->ntfs_client_id = o_msg-
>>> info.api_resp_info.param.init_rsp.client_id;
>> +    TRACE("Successfully recover client_id, new client_id:%u",
>> +                    client_hdl->ntfs_client_id);
>> +
>> +done:
>> +    if (o_msg)
>> +            ntfa_msg_destroy(o_msg);
>> +    TRACE_LEAVE();
>> +    return rc;
>> +}
>> +/****************************************************************
>> ************
>> +  Name          : recoverReader
>> +
>> +  Description   : This routine sends read_initialize messages to NTF
> Server in
>> +              order to reintroduce this reader to NTF Server
>> +
>> +  Arguments     : client_hdl* [IN]: client handle
>> +              reader_hdl* [IN]: reader handle
>> +
>> +  Return Values : SA_AIS_OK if success, others are failure
>> +
>> +  Notes         : None
>> +*****************************************************************
>> *************/
>> +SaAisErrorT recoverReader(ntfa_client_hdl_rec_t* client_hdl,
>> ntfa_reader_hdl_rec_t* reader_hdl) {
>> +    uint32_t mds_rc;
>> +    SaAisErrorT rc = SA_AIS_OK;
>> +    ntfsv_msg_t i_msg, *o_msg = NULL;
>> +    ntfsv_reader_init_req_2_t *send_param;
>> +
>> +    TRACE_ENTER();
>> +
>> +    /* Make sure the filters have not been deleted */
>> +    if (reader_hdl->filters.alarm_filter == NULL &&
>> +            reader_hdl->filters.att_ch_filter == NULL &&
>> +            reader_hdl->filters.obj_cr_del_filter == NULL &&
>> +            reader_hdl->filters.sec_al_filter == NULL &&
>> +            reader_hdl->filters.sta_ch_filter == NULL)
>> +            return SA_AIS_ERR_BAD_HANDLE;
>> +
>> +    memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>> +    i_msg.type = NTFSV_NTFA_API_MSG;
>> +    i_msg.info.api_info.type = NTFSV_READER_INITIALIZE_REQ_2;
>> +    send_param = &i_msg.info.api_info.param.reader_init_2;
>> +    send_param->head.client_id = client_hdl->ntfs_client_id;
>> +    send_param->head.searchCriteria = reader_hdl->searchCriteria;
>> +    send_param->f_rec = reader_hdl->filters;
>> +
>> +    mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>> NTFS_WAIT_TIME);
>> +
>> +    switch (mds_rc) {
>> +    case NCSCC_RC_SUCCESS:
>> +            break;
>> +    case NCSCC_RC_REQ_TIMOUT:
>> +            rc = SA_AIS_ERR_TRY_AGAIN;
>> +            goto done;
>> +    default:
>> +            rc = SA_AIS_ERR_BAD_HANDLE;
>> +            goto done;
>> +    }
>> +
>> +    osafassert(o_msg != NULL);
>> +    if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>> +            TRACE("o_msg->info.api_resp_info.rc:%u", o_msg-
>>> info.api_resp_info.rc);
>> +            rc = SA_AIS_ERR_BAD_HANDLE;
>> +            goto done;
>> +    }
>> +
>> +    if (o_msg->info.api_resp_info.type != NTFSV_READER_INITIALIZE_RSP) {
>> +            TRACE("msg type (%d) failed", (int)o_msg-
>>> info.api_resp_info.type);
>> +            rc = SA_AIS_ERR_LIBRARY;
>> +            goto done;
>> +    }
>> +    /* Update reader_id since it may be changed */
>> +    reader_hdl->reader_id = o_msg-
>>> info.api_resp_info.param.reader_init_rsp.readerId;
>> +
>> +    TRACE("Recover reader successfully");
>> +done:
>> +    if (o_msg)
>> +            ntfa_msg_destroy(o_msg);
>> +    TRACE_LEAVE();
>> +    return rc;
>> +}
>> +/****************************************************************
>> ************
>> +  Name          : recoverSubscriber
>> +
>> +  Description   : This routine sends subscribe messages to NTF Server in
>> +              order to reintroduce this subscriber to NTF Server
>> +
>> +  Arguments     : client_hdl* [IN]: client handle
>> +              subscriber_hdl* [IN]: subscriber handle
>> +
>> +  Return Values : SA_AIS_OK if success, others are failure
>> +
>> +  Notes         : None
>> +*****************************************************************
>> *************/
>> +SaAisErrorT recoverSubscriber(ntfa_client_hdl_rec_t* client_hdl,
>> +
>>      ntfa_subscriber_list_t* subscriber_hdl) {
>> +    uint32_t mds_rc;
>> +    SaAisErrorT rc = SA_AIS_OK;
>> +    ntfsv_msg_t i_msg, *o_msg = NULL;
>> +    ntfsv_subscribe_req_t *send_param;
>> +
>> +    TRACE_ENTER();
>> +
>> +    /* Make sure the filters have not been deleted */
>> +    if (subscriber_hdl->filters.alarm_filter == NULL &&
>> +            subscriber_hdl->filters.att_ch_filter == NULL &&
>> +            subscriber_hdl->filters.obj_cr_del_filter == NULL &&
>> +            subscriber_hdl->filters.sec_al_filter == NULL &&
>> +            subscriber_hdl->filters.sta_ch_filter == NULL)
>> +            return SA_AIS_ERR_BAD_HANDLE;
>> +
>> +    memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>> +    i_msg.type = NTFSV_NTFA_API_MSG;
>> +    i_msg.info.api_info.type = NTFSV_SUBSCRIBE_REQ;
>> +    send_param = &i_msg.info.api_info.param.subscribe;
>> +
>> +    send_param->client_id = client_hdl->ntfs_client_id;
>> +    send_param->subscriptionId = subscriber_hdl-
>>> subscriberListSubscriptionId;
>> +    send_param->f_rec = subscriber_hdl->filters;
>> +
>> +    mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>> NTFS_WAIT_TIME);
>> +
>> +    switch (mds_rc) {
>> +    case NCSCC_RC_SUCCESS:
>> +            break;
>> +    case NCSCC_RC_REQ_TIMOUT:
>> +            rc = SA_AIS_ERR_TRY_AGAIN;
>> +            goto done;
>> +    default:
>> +            rc = SA_AIS_ERR_BAD_HANDLE;
>> +            goto done;
>> +    }
>> +
>> +    osafassert(o_msg != NULL);
>> +
>> +    if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>> +            TRACE("o_msg->info.api_resp_info.rc:%u", o_msg-
>>> info.api_resp_info.rc);
>> +            rc = SA_AIS_ERR_BAD_HANDLE;
>> +            goto done;
>> +    }
>> +
>> +    if (o_msg->info.api_resp_info.type != NTFSV_SUBSCRIBE_RSP) {
>> +            TRACE("msg type (%d) failed", (int)o_msg-
>>> info.api_resp_info.type);
>> +            rc = SA_AIS_ERR_LIBRARY;
>> +            goto done;
>> +    }
>> +    TRACE("Recover subscriber successfully");
>> +done:
>> +    if (o_msg)
>> +            ntfa_msg_destroy(o_msg);
>> +
>> +    TRACE_LEAVE();
>> +    return rc;
>> +}
>> +/****************************************************************
>> ************
>> +  Name          : recoverClient
>> +
>> +  Description   : In NTF-A.03.01, section 2.1.2. A client can be producer,
>> +              subscriber, and reader at the same time.
>> +              This routine will recovery:
>> +              (1) client_id
>> +              (2) Then it will continue the recovery if this client also
>> +              has instance of subscriber or reader.
>> +  Arguments     : client_hdl* [IN]: client handle
>> +
>> +  Return Values : SA_AIS_OK if success, others are failure
>> +
>> +  Notes         : None
>> +*****************************************************************
>> *************/
>> +SaAisErrorT recoverClient(ntfa_client_hdl_rec_t *client_hdl) {
>> +
>> +    TRACE_ENTER();
>> +    SaAisErrorT rc = SA_AIS_OK;
>> +
>> +    osafassert(client_hdl);
>> +
>> +    if ((rc = reinitializeClient(client_hdl)) == SA_AIS_OK) {
>> +            /* Restore reader */
>> +            ntfa_reader_hdl_rec_t* reader_hdl = client_hdl->reader_list;
>> +            while (reader_hdl != NULL && rc == SA_AIS_OK) {
>> +                    rc = recoverReader(client_hdl, reader_hdl);
>> +                    reader_hdl = reader_hdl->next;
>> +            }
>> +            if (rc != SA_AIS_OK) {
>> +                    TRACE("Failed to restore reader (readerId:%d)",
>> +                                    reader_hdl->reader_id);
>> +                    goto done;
>> +            }
>> +            /* Restore subscriber */
>> +            ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
>> +            while (subscriber_hdl != NULL && rc == SA_AIS_OK) {
>> +                    if (client_hdl->local_hdl == subscriber_hdl-
>>> subscriberListNtfHandle)
>> +                            rc = recoverSubscriber(client_hdl,
>> subscriber_hdl);
>> +                    subscriber_hdl = subscriber_hdl->next;
>> +            }
>> +            if (rc != SA_AIS_OK) {
>> +                    TRACE("Failed to restore subscriber
>> (subscriptionId:%d)",
>> +                                    subscriber_hdl-
>>> subscriberListSubscriptionId);
>> +                    goto done;
>> +            }
>> +            client_hdl->valid = true;
>> +    } else {
>> +            TRACE("Failed to restore client (id:%d)", client_hdl-
>>> ntfs_client_id);
>> +            goto done;
>> +    }
>> +
>> +done:
>> +    TRACE_LEAVE();
>> +    return rc;
>> +}
>>
>> /*****************************************************************
>> **********
>>   * 8.4.1
>>   *
>> @@ -923,15 +1213,14 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>              version->releaseCode = NTF_RELEASE_CODE;
>>              version->majorVersion = NTF_MAJOR_VERSION;
>>              version->minorVersion = NTF_MINOR_VERSION;
>> -            ntfa_shutdown();
>> +            ntfa_shutdown(false);
>>              rc = SA_AIS_ERR_VERSION;
>>              goto done;
>>      }
>>
>> -    if (!ntfa_cb.ntfs_up) {
>> -            ntfa_shutdown();
>> -            TRACE("NTFS server is down");
>> -            rc = SA_AIS_ERR_TRY_AGAIN;
>> +    /* Check NTF server availability */
>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>> +            ntfa_shutdown(false);
>>              goto done;
>>      }
>>
>> @@ -977,7 +1266,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>              rc = SA_AIS_ERR_NO_MEMORY;
>>              goto err;
>>      }
>> -
>> +    ntfa_hdl_rec->version = *version;
>>      /* pass the handle value to the appl */
>>      if (SA_AIS_OK == rc)
>>              *ntfHandle = ntfa_hdl_rec->local_hdl;
>> @@ -990,7 +1279,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>
>>      if (rc != SA_AIS_OK) {
>>              TRACE_2("NTFA INIT FAILED\n");
>> -            ntfa_shutdown();
>> +            ntfa_shutdown(false);
>>      }
>>
>>   done:
>> @@ -1101,6 +1390,28 @@ SaAisErrorT saNtfDispatch(SaNtfHandleT n
>>              goto done;
>>      }
>>
>> +    /* Need to check NTF server availability here
>> +     * Because if the notificationCallback just comes after MDS_DOWN
>> +     * it will go to recovery then result in failure due to no director
>> +     */
>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>> +            ncshm_give_hdl(ntfHandle);
>> +            goto done;
>> +    }
>> +
>> +    if (!hdl_rec->valid) {
>> +            /* recovery */
>> +            if ((rc = recoverClient(hdl_rec)) != SA_AIS_OK) {
>> +                    if (rc == SA_AIS_ERR_BAD_HANDLE) {
>> +                            ncshm_give_hdl(ntfHandle);
>> +
>>      osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>> +                            ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
>> hdl_rec);
>> +
>>      osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>> +                            ntfa_shutdown(false);
>> +                            goto done;
>> +                    }
>> +            }
>> +    }
>>      if ((rc = ntfa_hdl_cbk_dispatch(&ntfa_cb, hdl_rec, dispatchFlags))
> !=
>> SA_AIS_OK)
>>              TRACE("NTFA_DISPATCH_FAILURE");
>>
>> @@ -1145,6 +1456,13 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>
>>      TRACE_ENTER();
>>
>> +    /* Check NTF server availability */
>> +    if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NO_ACTIVE) {
>> +            TRACE("NTFS server is temporarily unavailable");
>> +            rc = SA_AIS_ERR_TRY_AGAIN;
> [Vu] Better to unify using `checkNtfServerState()`?
[Minh] This case is special one, saNtfFinalize should be completed even 
if in headless
>> +            goto done;
>> +    }
>> +
>>      /* retrieve hdl rec */
>>      hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA, ntfHandle);
>>      if (hdl_rec == NULL) {
>> @@ -1152,43 +1470,37 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>              rc = SA_AIS_ERR_BAD_HANDLE;
>>              goto done;
>>      }
>> -
>> -    /* Check Whether NTFS is up or not */
>> -    if (!ntfa_cb.ntfs_up) {
>> -            TRACE("NTFS down");
>> -            rc = SA_AIS_ERR_TRY_AGAIN;
>> -            goto done_give_hdl;
>> +    if (hdl_rec->valid) {
>> +            /** populate & send the finalize message
>> +             ** and make sure the finalize from the server
>> +             ** end returned before deleting the local records.
>> +             **/
>> +            memset(&msg, 0, sizeof(ntfsv_msg_t));
>> +            msg.type = NTFSV_NTFA_API_MSG;
>> +            msg.info.api_info.type = NTFSV_FINALIZE_REQ;
>> +            msg.info.api_info.param.finalize.client_id = hdl_rec-
>>> ntfs_client_id;
>> +
>> +            mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>> NTFS_WAIT_TIME);
>> +            switch (mds_rc) {
>> +            case NCSCC_RC_SUCCESS:
>> +                    break;
>> +            case NCSCC_RC_REQ_TIMOUT:
>> +                    rc = SA_AIS_ERR_TIMEOUT;
>> +                    TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>> +                    goto done_give_hdl;
>> +            default:
>> +                    TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>> +                    rc = SA_AIS_ERR_NO_RESOURCES;
>> +                    goto done_give_hdl;
>> +            }
>> +
>> +            if (o_msg != NULL) {
>> +                    rc = o_msg->info.api_resp_info.rc;
>> +                    ntfa_msg_destroy(o_msg);
>> +            } else
>> +                    rc = SA_AIS_ERR_NO_RESOURCES;
>>      }
>>
>> -    /** populate & send the finalize message
>> -     ** and make sure the finalize from the server
>> -     ** end returned before deleting the local records.
>> -     **/
>> -    memset(&msg, 0, sizeof(ntfsv_msg_t));
>> -    msg.type = NTFSV_NTFA_API_MSG;
>> -    msg.info.api_info.type = NTFSV_FINALIZE_REQ;
>> -    msg.info.api_info.param.finalize.client_id =
> hdl_rec->ntfs_client_id;
>> -
>> -    mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>> NTFS_WAIT_TIME);
>> -    switch (mds_rc) {
>> -    case NCSCC_RC_SUCCESS:
>> -            break;
>> -    case NCSCC_RC_REQ_TIMOUT:
>> -            rc = SA_AIS_ERR_TIMEOUT;
>> -            TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>> -            goto done_give_hdl;
>> -    default:
>> -            TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>> -            rc = SA_AIS_ERR_NO_RESOURCES;
>> -            goto done_give_hdl;
>> -    }
>> -
>> -    if (o_msg != NULL) {
>> -            rc = o_msg->info.api_resp_info.rc;
>> -            ntfa_msg_destroy(o_msg);
>> -    } else
>> -            rc = SA_AIS_ERR_NO_RESOURCES;
>> -
>>      if (rc == SA_AIS_OK) {
>>      /** delete the hdl rec
>>           ** including all resources allocated by client if MDS send is
>> @@ -1205,7 +1517,7 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>      ncshm_give_hdl(ntfHandle);
>>
>>      if (rc == SA_AIS_OK) {
>> -            rc = ntfa_shutdown();
>> +            rc = ntfa_shutdown(false);
>>              if (rc != NCSCC_RC_SUCCESS)
>>                      TRACE_1("ntfa_shutdown failed");
>>      }
>> @@ -1349,7 +1661,7 @@ SaAisErrorT saNtfNotificationFree(SaNtfN
>>      }
>>
>>      /* free the resources allocated by
> saNtf<ntfType>NotificationAllocate
>> */
>> -    ntfa_hdl_rec_destructor(notification_hdl_rec);
>> +    ntfa_notification_destructor(notification_hdl_rec);
>>
>>      /** Delete the resources related to the notificationHandle &
>>       *  remove reference in the client.
>> @@ -1419,6 +1731,24 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>>      }
>>      osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>
>> +    /* Check NTF server availability */
>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK)
>> +            goto done_give_hdls;
>> +
>> +    /* Recover if this is invalid handle */
>> +    if (!client_rec->valid) {
>> +            if ((rc = recoverClient(client_rec)) != SA_AIS_OK) {
>> +                    ncshm_give_hdl(client_handle);
>> +                    ncshm_give_hdl(notificationHandle);
>> +                    if (rc == SA_AIS_ERR_BAD_HANDLE) {
>> +
>>      osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>> +                            ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
>> client_rec);
>> +
>>      osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>> +                            ntfa_shutdown(false);
>> +                    }
>> +                    goto err_free;
>> +            }
>> +    }
>>      /**
>>       ** Populate a sync MDS message
>>       **/
>> @@ -1481,13 +1811,6 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>>              goto done_give_hdls;
>>      }
>>
>> -    /* Check whether NTFS is up or not */
>> -    if (!ntfa_cb.ntfs_up) {
>> -            TRACE("NTFS down");
>> -            rc = SA_AIS_ERR_TRY_AGAIN;
>> -            goto done_give_hdls;
>> -    }
>> -
>>      /* Send a sync MDS message to obtain a notification id */
>>      mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>> timeout);
>>      switch (mds_rc) {
>> @@ -1596,7 +1919,8 @@ static  SaNtfHandleT ntfHandleGet(SaNtfS
>>      return ntfHandle;
>> }
>>
>> -static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
>> SaNtfSubscriptionIdT subscriptionId)
>> +static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
>> SaNtfSubscriptionIdT subscriptionId,
>> +
>>      ntfsv_filter_ptrs_t filters)
>> {
>>      SaAisErrorT rc = SA_AIS_OK;
>>      ntfa_subscriber_list_t* ntfSubscriberList;
>> @@ -1610,7 +1934,9 @@ static SaAisErrorT subscriptionListAdd(S
>>      /* Add ntfHandle and subscriptionId into list */
>>      ntfSubscriberList->subscriberListNtfHandle = ntfHandle;
>>      ntfSubscriberList->subscriberListSubscriptionId = subscriptionId;
>> -
>> +    memset(&ntfSubscriberList->filters, 0, sizeof(ntfsv_filter_ptrs_t));
>> +    ntfa_copy_ntf_filter_ptrs(&ntfSubscriberList->filters, &filters);
>> +
>>      osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>      if (NULL == subscriberNoList) {
>>              subscriberNoList = ntfSubscriberList;
>> @@ -1648,6 +1974,7 @@ static void subscriberListItemRemove(SaN
>>                      subscriberNoList = NULL;
>>      }
>>      TRACE_1("REMOVE: listPtr->SubscriptionId %d", listPtr-
>>> subscriberListSubscriptionId);
>> +    ntfa_del_ntf_filter_ptrs(&listPtr->filters);
>>      free(listPtr);
>>      osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>> }
>> @@ -1666,20 +1993,33 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>      ntfsv_msg_t msg, *o_msg = NULL;
>>      ntfsv_subscribe_req_t *send_param;
>>      uint32_t timeout = NTFS_WAIT_TIME;
>> -
>> +    bool recovery_failed = false;
>>      TRACE_ENTER();
>> +
>>      rc = getFilters(notificationFilterHandles, &filters, &ntfHandle,
>> &client_hdl_rec);
>>      if (rc != SA_AIS_OK) {
>>              TRACE("getFilters failed");
>>              goto done;
>>      }
>>
>> +    /* Check NTF server availability */
>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK)
>> +            goto done;
>> +
>> +    /* recovery */
>> +    if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
>> +            if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>> +                    recovery_failed = true;
>> +                    goto done;
>> +            }
>> +    }
>> +
>>      tmpHandle = ntfHandleGet(subscriptionId);
>>      if (tmpHandle != 0) {
>>              rc = SA_AIS_ERR_EXIST;
>>              goto done;
>>      }
>> -    rc = subscriptionListAdd(ntfHandle, subscriptionId);
>> +    rc = subscriptionListAdd(ntfHandle, subscriptionId, filters);
>>      if (rc != SA_AIS_OK) {
>>              goto done;
>>      }
>> @@ -1693,29 +2033,24 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>      send_param->client_id = client_hdl_rec->ntfs_client_id;
>>      send_param->subscriptionId = subscriptionId;
>>      send_param->f_rec = filters;
>> -    /* Check whether NTFS is up or not */
>> -    if (ntfa_cb.ntfs_up) {
>> -            uint32_t rv;
>> -            /* Send a sync MDS message to obtain a log stream id */
>> -            rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>> timeout);
>> -            if (rv == NCSCC_RC_SUCCESS) {
>> -                    osafassert(o_msg != NULL);
>> -                    if (SA_AIS_OK == o_msg->info.api_resp_info.rc) {
>> -                            TRACE_1("subscriptionId from server %u",
>> -                                    o_msg-
>>> info.api_resp_info.param.subscribe_rsp.subscriptionId);
>> -                    } else {
>> -                            rc = o_msg->info.api_resp_info.rc;
>> -                            TRACE("Bad return status!!! rc = %d", rc);
>> -                    }
>> +
>> +    uint32_t rv;
>> +    /* Send a sync MDS message to obtain a log stream id */
>> +    rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>> +    if (rv == NCSCC_RC_SUCCESS) {
>> +            osafassert(o_msg != NULL);
>> +            if (SA_AIS_OK == o_msg->info.api_resp_info.rc) {
>> +                    TRACE_1("subscriptionId from server %u",
>> +                            o_msg-
>>> info.api_resp_info.param.subscribe_rsp.subscriptionId);
>>              } else {
>> -                    if(rv == NCSCC_RC_INVALID_INPUT)
>> -                            rc = SA_AIS_ERR_INVALID_PARAM;
>> -                    else
>> -                            rc = SA_AIS_ERR_TRY_AGAIN;
>> +                    rc = o_msg->info.api_resp_info.rc;
>> +                    TRACE("Bad return status!!! rc = %d", rc);
>>              }
>>      } else {
>> -            TRACE_1("NTFS down");
>> -            rc = SA_AIS_ERR_TRY_AGAIN;
>> +            if(rv == NCSCC_RC_INVALID_INPUT)
>> +                    rc = SA_AIS_ERR_INVALID_PARAM;
>> +            else
>> +                    rc = SA_AIS_ERR_TRY_AGAIN;
>>      }
>>
> [Vu] Seems missing SA_AIS_ERR_TIMEOUT err code here.
[Minh] Yes it seems to be, but it's same as code base. I think it should 
be addressed in different ticket to scan through all APIs
>>      if (rc != SA_AIS_OK) {
>> @@ -1724,6 +2059,7 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>      if (o_msg)
>>              ntfa_msg_destroy(o_msg);
>>      done:
>> +
>>      ncshm_give_hdl(ntfHandle);
>>      if (notificationFilterHandles) {
>>              if (notificationFilterHandles->attributeChangeFilterHandle)
>> @@ -1737,6 +2073,13 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>              if (notificationFilterHandles->alarmFilterHandle)
>>                      ncshm_give_hdl(notificationFilterHandles-
>>> alarmFilterHandle);
>>      }
>> +    if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
>> +            osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>> +            ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
> client_hdl_rec);
>> +            osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>> +            ntfa_shutdown(false);
>> +    }
>> +
>>      TRACE_LEAVE();
>>      return rc;
>> }
>> @@ -1995,6 +2338,7 @@ SaAisErrorT saNtfSecurityAlarmNotificati
>>              rc = SA_AIS_ERR_INVALID_PARAM;
>>              goto done;
>>      }
>> +
>>      /* retrieve hdl rec */
>>      hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA, ntfHandle);
>>      if (hdl_rec == NULL) {
>> @@ -2058,6 +2402,7 @@ SaAisErrorT saNtfPtrValAllocate(SaNtfNot
>>              rc = SA_AIS_ERR_INVALID_PARAM;
>>              goto done;
>>      }
>> +
>>      notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>> notificationHandle);
>>      if (notification_hdl_rec == NULL) {
>>              TRACE("ncshm_take_hdl notificationHandle failed");
>> @@ -2098,6 +2443,7 @@ SaAisErrorT saNtfArrayValAllocate(SaNtfN
>>              rc = SA_AIS_ERR_INVALID_PARAM;
>>              goto done;
>>      }
>> +
>>      notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>> notificationHandle);
>>      if (notification_hdl_rec == NULL) {
>>              TRACE("ncshm_take_hdl notificationHandle failed");
>> @@ -2228,6 +2574,7 @@ SaAisErrorT saNtfArrayValGet(SaNtfNotifi
>>              rc = SA_AIS_ERR_INVALID_PARAM;
>>              goto done;
>>      }
>> +
>>      notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>> notificationHandle);
>>      if (notification_hdl_rec == NULL) {
>>              TRACE("ncshm_take_hdl notificationHandle failed");
>> @@ -2695,7 +3042,7 @@ SaAisErrorT saNtfNotificationFilterFree(
>>      }
>>
>>      /* free the resources allocated by saNtf<ntfType>FilterAllocate */
>> -    ntfa_filter_hdl_rec_destructor(filter_hdl_rec);
>> +    ntfa_filter_destructor(filter_hdl_rec);
>>
>>      /** Delete the resources related to the notificationFilterHandle &
>>       *  remove reference in the client.
>> @@ -2729,7 +3076,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>      TRACE_ENTER();
>>      SaAisErrorT rc = SA_AIS_ERR_NOT_EXIST;
>>      SaNtfHandleT ntfHandle;
>> -
>>      ntfa_client_hdl_rec_t *client_hdl_rec;
>>
>>      ntfsv_msg_t msg, *o_msg = NULL;
>> @@ -2737,7 +3083,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>      ntfsv_unsubscribe_req_t *send_param;
>>      uint32_t timeout = NTFS_WAIT_TIME;
>>
>> -
>>      ntfHandle = ntfHandleGet(subscriptionId);
>>      if (ntfHandle == 0) {
>>              TRACE_1("ntfHandleGet failed, subscription not exist");
>> @@ -2753,9 +3098,16 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>              goto done;
>>      }
>>
>> -    /**
>> -         ** Populate a sync MDS message
>> -         **/
>> +    /* Check NTF server availability */
>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>> +            ncshm_give_hdl(ntfHandle);
>> +            goto done;
>> +    }
>> +
>> +    if (client_hdl_rec->valid) {
>> +            /**
>> +             ** Populate a sync MDS message
>> +             **/
>>              memset(&msg, 0, sizeof(ntfsv_msg_t));
>>              msg.type = NTFSV_NTFA_API_MSG;
>>              msg.info.api_info.type = NTFSV_UNSUBSCRIBE_REQ;
>> @@ -2764,13 +3116,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>              send_param->client_id = client_hdl_rec->ntfs_client_id;
>>              send_param->subscriptionId = subscriptionId;
>>
>> -            /* Check whether NTFS is up or not */
>> -            if (!ntfa_cb.ntfs_up) {
>> -                    TRACE_1("NTFS down");
>> -                    rc = SA_AIS_ERR_TRY_AGAIN;
>> -                    goto done_give_hdl;
>> -            }
>> -
>>              /* Send a sync MDS message to obtain a log stream id */
>>              rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>> timeout);
>>              if (rc != NCSCC_RC_SUCCESS) {
>> @@ -2785,6 +3130,7 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>                      TRACE_1("Bad return status! rc = %d", rc);
>>                      goto done_give_hdl;
>>              }
>> +    }
>>              subscriberListItemRemove(subscriptionId);
>>
>>              /*Remove msg for subscriptionId from mailbox*/
>> @@ -2815,6 +3161,19 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>   done_give_hdl:
>>      if (o_msg)
>>               ntfa_msg_destroy(o_msg);
>> +
>> +    if (!client_hdl_rec->valid) {
>> +            if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>> +                    if (rc == SA_AIS_ERR_BAD_HANDLE) {
>> +                            ncshm_give_hdl(ntfHandle);
>> +
>>      osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>> +                            ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
>> client_hdl_rec);
>> +
>>      osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>> +                            ntfa_shutdown(false);
>> +                            goto done;
>> +                    }
>> +            }
>> +    }
>>      ncshm_give_hdl(ntfHandle);
>>   done:
>>      TRACE_LEAVE();
>> @@ -2838,6 +3197,7 @@ SaAisErrorT saNtfNotificationReadInitial
>>      ntfsv_reader_init_req_2_t *send_param;
>>      uint32_t timeout = NTFS_WAIT_TIME;
>>
>> +    bool recovery_failed = false;
>>      TRACE_ENTER();
>>      if (notificationFilterHandles == NULL || readHandle == NULL) {
>>              rc = SA_AIS_ERR_INVALID_PARAM;
>> @@ -2872,6 +3232,18 @@ SaAisErrorT saNtfNotificationReadInitial
>>              goto done_give_client_hdl;
>>      }
>>
>> +    /* Check NTF server availability */
>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK)
>> +            goto done_give_client_hdl;
>> +
>> +    /* recovery */
>> +    if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
>> +            if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>> +                    recovery_failed = true;
>> +                    goto done_give_client_hdl;
>> +            }
>> +    }
>> +
>>      /**
>>       ** Populate a sync MDS message
>>       **/
>> @@ -2884,13 +3256,6 @@ SaAisErrorT saNtfNotificationReadInitial
>>      send_param->head.searchCriteria = searchCriteria;
>>      send_param->f_rec = filters;
>>
>> -    /* Check whether NTFS is up or not */
>> -    if (!ntfa_cb.ntfs_up) {
>> -            TRACE("NTFS down");
>> -            rc = SA_AIS_ERR_TRY_AGAIN;
>> -            goto done_give_client_hdl;
>> -    }
>> -
>>      /* Send a sync MDS message to obtain a log stream id */
>>      rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>      if (rc != NCSCC_RC_SUCCESS) {
>> @@ -2925,12 +3290,16 @@ SaAisErrorT saNtfNotificationReadInitial
>>      reader_hdl_rec->ntfHandle = ntfHandle;
>>      /* Store the readerId returned from server */
>>      reader_hdl_rec->reader_id = o_msg-
>>> info.api_resp_info.param.reader_init_rsp.readerId;
>> +    memset(&reader_hdl_rec->filters, 0, sizeof(ntfsv_filter_ptrs_t));
>> +    ntfa_copy_ntf_filter_ptrs(&reader_hdl_rec->filters, &filters);
>> +    reader_hdl_rec->searchCriteria = searchCriteria;
>>      /**                  UnLock ntfa_CB            **/
>>      osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>
>> done_give_client_hdl:
>>      if (o_msg)
>>              ntfa_msg_destroy(o_msg);
>> +
>>     if (client_hdl_rec)
>>              ncshm_give_hdl(client_hdl_rec->local_hdl);
>>      if (notificationFilterHandles) {
>> @@ -2947,6 +3316,12 @@ done_give_client_hdl:
>>      }
>>
>>      ncshm_give_hdl(notificationFilterHandles->alarmFilterHandle);
>> +    if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
>> +            osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>> +            ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
> client_hdl_rec);
>> +            osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>> +            ntfa_shutdown(false);
>> +    }
>> done:
>>      TRACE_LEAVE();
>>      return rc;
>> @@ -2983,42 +3358,46 @@ SaAisErrorT saNtfNotificationReadFinaliz
>>      }
>>      TRACE_1("reader_hdl_rec = %u", reader_hdl_rec->reader_hdl);
>>
>> -    /**
>> -     ** Populate a sync MDS message
>> -     **/
>> -    memset(&msg, 0, sizeof(ntfsv_msg_t));
>> -    msg.type = NTFSV_NTFA_API_MSG;
>> -    msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
>> -    send_param = &msg.info.api_info.param.reader_finalize;
>> -    send_param->client_id = client_hdl_rec->ntfs_client_id;
>> -    send_param->readerId = reader_hdl_rec->reader_id;
>> -
>> -    /* Check whether NTFS is up or not */
>> -    if (!ntfa_cb.ntfs_up) {
>> -            TRACE("NTFS down");
>> -            rc = SA_AIS_ERR_TRY_AGAIN;
>> -            goto done_give_hdls;
>> +    /* Check NTF server availability */
>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>> +            ncshm_give_hdl(client_hdl_rec->local_hdl);
>> +            ncshm_give_hdl(readhandle);
>> +            goto done;
>>      }
>>
>> -    /* Send a sync MDS message */
>> -    rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>> -    if (rc != NCSCC_RC_SUCCESS) {
>> -            rc = SA_AIS_ERR_TRY_AGAIN;
>> -            goto done_give_hdls;
>> +    if (client_hdl_rec->valid) {
>> +            /**
>> +             ** Populate a sync MDS message
>> +             **/
>> +            memset(&msg, 0, sizeof(ntfsv_msg_t));
>> +            msg.type = NTFSV_NTFA_API_MSG;
>> +            msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
>> +            send_param = &msg.info.api_info.param.reader_finalize;
>> +            send_param->client_id = client_hdl_rec->ntfs_client_id;
>> +            send_param->readerId = reader_hdl_rec->reader_id;
>> +
>> +
>> +            /* Send a sync MDS message */
>> +            rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>> timeout);
>> +            if (rc != NCSCC_RC_SUCCESS) {
>> +                    rc = SA_AIS_ERR_TRY_AGAIN;
>> +                    goto done_give_hdls;
>> +            }
>> +
>> +            osafassert(o_msg != NULL);
>> +            if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
>> +                    rc = o_msg->info.api_resp_info.rc;
>> +                    TRACE("Bad return status!!! rc = %d", rc);
>> +                    goto done_give_hdls;
>> +            }
>> +
>> +            if (o_msg->info.api_resp_info.type !=
>> NTFSV_READER_FINALIZE_RSP) {
>> +                    TRACE("msg type (%d) failed", (int)o_msg-
>>> info.api_resp_info.type);
>> +                    rc = SA_AIS_ERR_LIBRARY;
>> +                    goto done_give_hdls;
>> +            }
>>      }
>>
>> -    osafassert(o_msg != NULL);
>> -    if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
>> -            rc = o_msg->info.api_resp_info.rc;
>> -            TRACE("Bad return status!!! rc = %d", rc);
>> -            goto done_give_hdls;
>> -    }
>> -
>> -    if (o_msg->info.api_resp_info.type != NTFSV_READER_FINALIZE_RSP) {
>> -            TRACE("msg type (%d) failed", (int)o_msg-
>>> info.api_resp_info.type);
>> -            rc = SA_AIS_ERR_LIBRARY;
>> -            goto done_give_hdls;
>> -    }
>>      osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>      oas_rc = ntfa_reader_hdl_rec_del(&client_hdl_rec->reader_list,
>> reader_hdl_rec);
>>      if (oas_rc != NCSCC_RC_SUCCESS) {
>> @@ -3029,6 +3408,20 @@ SaAisErrorT saNtfNotificationReadFinaliz
>>   done_give_hdls:
>>      if (o_msg)
>>               ntfa_msg_destroy(o_msg);
>> +
>> +    if (!client_hdl_rec->valid) {
>> +            if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>> +                    if (rc == SA_AIS_ERR_BAD_HANDLE) {
>> +                            ncshm_give_hdl(client_hdl_rec->local_hdl);
>> +                            ncshm_give_hdl(readhandle);
>> +
>>      osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>> +                            ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
>> client_hdl_rec);
>> +
>>      osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>> +                            ntfa_shutdown(false);
>> +                            goto done;
>> +                    }
>> +            }
>> +    }
>>      ncshm_give_hdl(client_hdl_rec->local_hdl);
>>   done_give_read_hdl:
>>      ncshm_give_hdl(readhandle);
>> @@ -3082,6 +3475,23 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>>      }
>>      TRACE_1("reader_hdl_rec = %u", reader_hdl_rec->reader_hdl);
>>
>> +    /* Check NTF server availability */
>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK)
>> +            goto done_give_hdls;
>> +
>> +    if (!client_hdl_rec->valid) {
>> +            if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>> +                    ncshm_give_hdl(client_hdl_rec->local_hdl);
>> +                    ncshm_give_hdl(readHandle);
>> +                    if (rc == SA_AIS_ERR_BAD_HANDLE) {
>> +
>>      osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>> +                            ntfa_hdl_rec_force_del(&ntfa_cb.client_list,
>> client_hdl_rec);
>> +
>>      osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>> +                            ntfa_shutdown(false);
>> +                    }
>> +                    goto done;
>> +            }
>> +    }
>>      /**
>>       ** Populate a sync MDS message
>>       **/
>> @@ -3092,12 +3502,6 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>>      send_param->client_id = client_hdl_rec->ntfs_client_id;
>>      send_param->readerId = reader_hdl_rec->reader_id;
>>      send_param->searchDirection = searchDirection;
>> -    /* Check whether NTFS is up or not */
>> -    if (!ntfa_cb.ntfs_up) {
>> -            TRACE("NTFS down");
>> -            rc = SA_AIS_ERR_TRY_AGAIN;
>> -            goto done_give_hdls;
>> -    }
>>
>>      do {
>>              /* Send a sync MDS message */
>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>> b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>> --- a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>> @@ -369,7 +369,8 @@ static uint32_t ntfa_mds_svc_evt(struct
>>                      TRACE("NTFS down");
>>                      pthread_mutex_lock(&ntfa_cb.cb_lock);
>>                      memset(&ntfa_cb.ntfs_mds_dest, 0,
>> sizeof(MDS_DEST));
>> -                    ntfa_cb.ntfs_up = 0;
>> +                    ntfa_update_ntfsv_state(mds_cb_info-
>>> info.svc_evt.i_change == NCSMDS_NO_ACTIVE ?
>> +                                    NTFA_NTFSV_NO_ACTIVE :
>> NTFA_NTFSV_DOWN);
>>                      pthread_mutex_unlock(&ntfa_cb.cb_lock);
>>              }
>>              break;
>> @@ -382,7 +383,8 @@ static uint32_t ntfa_mds_svc_evt(struct
>>                      TRACE_2("MSG from NTFS NCSMDS_NEW_ACTIVE/UP");
>>                      pthread_mutex_lock(&ntfa_cb.cb_lock);
>>                      ntfa_cb.ntfs_mds_dest = mds_cb_info-
>>> info.svc_evt.i_dest;
>> -                    ntfa_cb.ntfs_up = 1;
>> +                    ntfa_update_ntfsv_state(mds_cb_info-
>>> info.svc_evt.i_change == NCSMDS_NEW_ACTIVE ?
>> +                                    NTFA_NTFSV_NEW_ACTIVE :
>> NTFA_NTFSV_UP);
>>                      if (ntfa_cb.ntfs_sync_awaited) {
>>                              /* signal waiting thread */
>>                              m_NCS_SEL_OBJ_IND(&ntfa_cb.ntfs_sync_sel);
>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_util.c
>> b/osaf/libs/agents/saf/ntfa/ntfa_util.c
>> --- a/osaf/libs/agents/saf/ntfa/ntfa_util.c
>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_util.c
>> @@ -49,7 +49,7 @@ static unsigned int ntfa_create(void)
>>      }
>>
>>      /* Block and wait for indication from MDS meaning NTFS is up */
>> -    osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_sync_sel),
>> 30000);
>> +    osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_sync_sel),
>> 10000);
>>
>>      pthread_mutex_lock(&ntfa_cb.cb_lock);
>>      ntfa_cb.ntfs_sync_awaited = 0;
>> @@ -108,7 +108,25 @@ static bool ntfa_clear_mbx(NCSCONTEXT ar
>>      }
>>      return true;
>> }
>> +/****************************************************************
>> ************
>> +  Name          : ntfa_notification_list_del
>>
>> +  Description   : This routine free element allocated in list of
> notification
>> +
>> +  Arguments     : pointer to the list of notification records anchor.
>> +
>> +  Return Values : None
>> +
>> +  Notes         :
>> +*****************************************************************
>> *************/
>> +static void ntfa_notification_list_del(ntfa_notification_hdl_rec_t
> **plstr_hdl)
>> +{
>> +    ntfa_notification_hdl_rec_t *lstr_hdl = *plstr_hdl;
>> +    while (lstr_hdl != NULL) {
>> +            ntfa_notification_destructor(lstr_hdl);
>> +            lstr_hdl = lstr_hdl->next;
>> +    }
>> +}
>>
>> /*****************************************************************
>> ***********
>>    Name          : ntfa_notification_hdl_rec_list_del
>>
>> @@ -130,6 +148,70 @@ static void ntfa_notification_hdl_rec_li
>>              lstr_hdl = NULL;
>>      }
>> }
>> +/****************************************************************
>> ************
>> +  Name          : ntfa_filter_list_del
>> +
>> +  Description   : This routine free element allocated in list of filter
>> +
>> +  Arguments     : pointer to the list of filters records anchor.
>> +
>> +  Return Values : None
>> +
>> +  Notes         :
>> +*****************************************************************
>> *************/
>> +static void ntfa_filter_list_del(ntfa_filter_hdl_rec_t **plstr_hdl)
>> +{
>> +    ntfa_filter_hdl_rec_t *lstr_hdl = *plstr_hdl;
>> +
>> +    while (lstr_hdl != NULL) {
>> +            ntfa_filter_destructor(lstr_hdl);
>> +            lstr_hdl = lstr_hdl->next;
>> +    }
>> +}
>> +/****************************************************************
>> ************
>> +  Name          : ntfa_filter_hdl_rec_list_del
>> +
>> +  Description   : This routine deletes a list of allocated filters
>> +
>> +  Arguments     : pointer to the list of filters records anchor.
>> +
>> +  Return Values : None
>> +
>> +  Notes         :
>> +*****************************************************************
>> *************/
>> +static void ntfa_filter_hdl_rec_list_del(ntfa_filter_hdl_rec_t
> **plstr_hdl)
>> +{
>> +    ntfa_filter_hdl_rec_t *lstr_hdl;
>> +
>> +    while ((lstr_hdl = *plstr_hdl) != NULL) {
>> +            *plstr_hdl = lstr_hdl->next;
>> +            ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, lstr_hdl-
>>> filter_hdl);
>> +            free(lstr_hdl);
>> +            lstr_hdl = NULL;
>> +    }
>> +}
>> +/****************************************************************
>> ************
>> +  Name          : ntfa_reader_hdl_rec_list_del
>> +
>> +  Description   : This routine deletes a list of allocated readers
>> +
>> +  Arguments     : pointer to the list of readers records anchor.
>> +
>> +  Return Values : None
>> +
>> +  Notes         :
>> +*****************************************************************
>> *************/
>> +static void ntfa_reader_hdl_rec_list_del(ntfa_reader_hdl_rec_t
> **plstr_hdl)
>> +{
>> +    ntfa_reader_hdl_rec_t *lstr_hdl;
>> +    while ((lstr_hdl = *plstr_hdl) != NULL) {
>> +            *plstr_hdl = lstr_hdl->next;
>> +            ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, lstr_hdl-
>>> reader_hdl);
>> +            ntfa_del_ntf_filter_ptrs(&lstr_hdl->filters);
>> +            free(lstr_hdl);
>> +            lstr_hdl = NULL;
>> +    }
>> +}
>>
>> static SaAisErrorT ntfa_alloc_callback_notification(SaNtfNotificationsT
>> *notification,
>>                                                  ntfsv_send_not_req_t
>> *not_cbk, ntfa_client_hdl_rec_t *hdl_rec)
>> @@ -407,6 +489,10 @@ static SaAisErrorT ntfa_hdl_cbk_rec_prc(
>>                      free(cbk_info-
>>> param.discarded_cbk.discardedNotificationIdentifiers);
>>              }
>>              break;
>> +    case NTFSV_DUMMY_CALLBACK:
>> +            TRACE("Do nothing with dummy callback, just return OK");
>> +            rc = SA_AIS_OK;
>> +            break;
>>      default:
>>              TRACE("unsupported callback type: %d", cbk_info->type);
>>              rc = SA_AIS_ERR_LIBRARY;
>> @@ -544,20 +630,21 @@ unsigned int ntfa_startup(void)
>>   *
>>   * @return unsigned int
>>   */
>> -unsigned int ntfa_shutdown(void)
>> +unsigned int ntfa_shutdown(bool forced)
>> {
>>      unsigned int rc = NCSCC_RC_SUCCESS;
>>
>> -    TRACE_ENTER2("ntfa_use_count: %u", ntfa_use_count);
>> +    TRACE_ENTER2("ntfa_use_count: %u, forced: %u", ntfa_use_count,
>> forced);
>>      pthread_mutex_lock(&ntfa_lock);
>>
>> -    if (ntfa_use_count > 1) {
>> -            /* Users still exist, just decrement the use count */
>> -            ntfa_use_count--;
>> -    } else if (ntfa_use_count == 1) {
>> +    if ((forced && (ntfa_use_count > 0)) || (ntfa_use_count == 1)) {
>>              ntfa_destroy();
>>              rc = ncs_agents_shutdown();
>>              ntfa_use_count = 0;
>> +            ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_NONE;
>> +    } else if (ntfa_use_count > 1) {
>> +            /* Users still exist, just decrement the use count */
>> +            ntfa_use_count--;
>>      }
>>
>>      pthread_mutex_unlock(&ntfa_lock);
>> @@ -633,7 +720,27 @@ ntfa_client_hdl_rec_t *ntfa_find_hdl_rec
>>
>>      return NULL;
>> }
>> +/****************************************************************
>> ************
>> +  Name          : ntfa_subscriber_list_del
>>
>> +  Description   : This routine deletes a list of allocated subscribers
>> +
>> +  Arguments     : pointer to the list of subscribers records anchor.
>> +
>> +  Return Values : None
>> +
>> +  Notes         :
>> +*****************************************************************
>> *************/
>> +void ntfa_subscriber_list_del()
>> +{
>> +    ntfa_subscriber_list_t *listPtr = subscriberNoList;
>> +    while (listPtr != NULL) {
>> +            ntfa_subscriber_list_t* tmpSub = listPtr;
>> +            listPtr = listPtr->next;
>> +            free(tmpSub);
>> +    }
>> +    subscriberNoList = NULL;
>> +}
>>
>> /*****************************************************************
>> ***********
>>    Name          : ntfa_hdl_list_del
>>
>> @@ -651,15 +758,24 @@ void ntfa_hdl_list_del(ntfa_client_hdl_r
>>
>>      while ((client_hdl = *p_client_hdl) != NULL) {
>>              *p_client_hdl = client_hdl->next;
>> +            m_NCS_IPC_DETACH(&client_hdl->mbx, ntfa_clear_mbx, NULL);
>> +            m_NCS_IPC_RELEASE(&client_hdl->mbx, NULL);
>>              ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, client_hdl-
>>> local_hdl);
>>      /** clean up the channel records for this ntfa-client
>>           **/
>> +            ntfa_notification_list_del(&client_hdl->notification_list);
>>              ntfa_notification_hdl_rec_list_del(&client_hdl-
>>> notification_list);
>> +
>> +            ntfa_filter_list_del(&client_hdl->filter_list);
>> +            ntfa_filter_hdl_rec_list_del(&client_hdl->filter_list);
>> +
>> +            ntfa_reader_hdl_rec_list_del(&client_hdl->reader_list);
>>      /** remove the association with hdl-mngr
>>           **/
>>              free(client_hdl);
>>              client_hdl = 0;
>>      }
>> +    ntfa_subscriber_list_del();
>> }
>>
>>
>> /*****************************************************************
>> ***********
>> @@ -757,10 +873,80 @@ uint32_t ntfa_filter_hdl_rec_del(ntfa_fi
>>      TRACE("The node couldn't be deleted");
>>      return NCSCC_RC_FAILURE;
>> }
>> +/****************************************************************
>> ************
>> +  Name          : ntfa_hdl_rec_force_del
>>
>> +  Description   : This routine deletes all memory allocated to client.
>> +
>> +  Arguments     : NTFA_CLIENT_HDL_REC **list_head
>> +              NTFA_CLIENT_HDL_REC *rm_node
>> +
>> +  Return Values : None
>> +
>> +  Notes         :
>> +*****************************************************************
>> *************/
>> +void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **list_head,
>> ntfa_client_hdl_rec_t *rm_node)
>> +{
>> +    ntfa_client_hdl_rec_t *list_iter = *list_head;
>> +    TRACE_ENTER();
>> +    /* First remove the rm_node out of the list of client */
>> +    if (list_iter == rm_node)
>> +            *list_head = rm_node->next;
>> +    else {
>> +            while (list_iter) {
>> +                    if (list_iter->next == rm_node) {
>> +                            list_iter->next = rm_node->next;
>> +                            break;
>> +                    }
>> +                    list_iter = list_iter->next;
>> +            }
>> +    }
>> +    /* Release all msgs in mailbox */
>> +    ntfsv_msg_t *cbk_msg;
>> +    while((cbk_msg =
>> (ntfsv_msg_t*)m_NCS_IPC_NON_BLK_RECEIVE(&rm_node->mbx, cbk_msg))
>> +            != NULL) {
>> +            ntfa_msg_destroy(cbk_msg);
>> +    }
>> +    /* delete subscriber of this client out of the subcriberNoList*/
>> +    ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
>> +    while (subscriber_hdl != NULL) {
>> +            ntfa_subscriber_list_t *rm_subscriber = subscriber_hdl;
>> +            subscriber_hdl = subscriber_hdl->next;
>> +            if (rm_node->local_hdl == rm_subscriber-
>>> subscriberListNtfHandle) {
>> +                    if (rm_subscriber->next != NULL) {
>> +                            rm_subscriber->next->prev = rm_subscriber-
>>> prev;
>> +                    }
>> +
>> +                    if (rm_subscriber->prev != NULL) {
>> +                            rm_subscriber->prev->next = rm_subscriber-
>>> next;
>> +                    } else {
>> +                            if (rm_subscriber->next != NULL)
>> +                                    subscriberNoList = rm_subscriber-
>>> next;
>> +                            else
>> +                                    subscriberNoList = NULL;
>> +                    }
>> +                    ntfa_del_ntf_filter_ptrs(&rm_subscriber->filters);
>> +                    free(rm_subscriber);
>> +            }
>> +    }
>> +    /* Now delete client */
>> +    m_NCS_IPC_DETACH(&rm_node->mbx, ntfa_clear_mbx, NULL);
>> +    m_NCS_IPC_RELEASE(&rm_node->mbx, NULL);
>> +    ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node->local_hdl);
>> +    ntfa_notification_list_del(&rm_node->notification_list);
>> +    ntfa_notification_hdl_rec_list_del(&rm_node->notification_list);
>> +
>> +    ntfa_filter_list_del(&rm_node->filter_list);
>> +    ntfa_filter_hdl_rec_list_del(&rm_node->filter_list);
>> +
>> +    ntfa_reader_hdl_rec_list_del(&rm_node->reader_list);
>> +    free(rm_node);
>> +
>> +    TRACE_LEAVE();
>> +}
>>
>> /*****************************************************************
>> ***********
>>    Name          : ntfa_hdl_rec_del
>> -
>> +
>>    Description   : This routine deletes the a client handle record from
>>                    a list of client hdl records.
>>
>> @@ -962,7 +1148,7 @@ ntfa_client_hdl_rec_t *ntfa_hdl_rec_add(
>>      /** Associate with the client_id obtained from NTFS
>>       **/
>>      rec->ntfs_client_id = client_id;
>> -
>> +    rec->valid = true;
>>      /** Initialize and attach the IPC/Priority queue
>>       **/
>>
>> @@ -1063,7 +1249,7 @@ static void logtrace_init_constructor(vo
>>   *
>>   * @param instance
>>   */
>> -void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t *instance)
>> +void ntfa_notification_destructor(ntfa_notification_hdl_rec_t *instance)
>> {
>>      ntfa_notification_hdl_rec_t *notificationInstance = instance;
>>
>> @@ -1111,7 +1297,7 @@ void ntfa_hdl_rec_destructor(ntfa_notifi
>>   *
>>   * @param instance
>>   */
>> -void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t *filter_rec)
>> +void ntfa_filter_destructor(ntfa_filter_hdl_rec_t *filter_rec)
>> {
>>      switch (filter_rec->ntfType) {
>>      case SA_NTF_TYPE_OBJECT_CREATE_DELETE:
>> @@ -1202,6 +1388,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>>           **/
>>              ncshm_give_hdl(rm_node->reader_hdl);
>>              ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node-
>>> reader_hdl);
>> +            ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>>              free(rm_node);
>>              return NCSCC_RC_SUCCESS;
>>      } else {                /* find the rec */
>> @@ -1213,6 +1400,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>>                   **/
>>                              ncshm_give_hdl(rm_node->reader_hdl);
>>                              ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA,
>> rm_node->reader_hdl);
>> +                            ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>>                              free(rm_node);
>>                              return NCSCC_RC_SUCCESS;
>>                      }
>> @@ -1258,3 +1446,258 @@ void ntfa_add_to_async_cbk_msg_list(ntfs
>>
>>      TRACE_LEAVE();
>> }
>> +
>> +/****************************************************************
>> ************
>> +  Name          : ntfa_notify_handle_invalid
>> +
>> +  Description   : This routine sends a dummy callback msg to client's
> mailbox
>> +              so that the client polls in and calls saNtfDispatch.
>> +  Arguments     :
>> +
>> +  Return Values : None
>> +
>> +  Notes         : None
>> +*****************************************************************
>> *************/
>> +void ntfa_notify_handle_invalid() {
>> +    ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
>> +    TRACE_ENTER();
>> +    while (client_hdl != NULL) {
>> +            /* Only applicable for subscriber */
>> +            if (client_hdl->reg_cbk.saNtfNotificationCallback != NULL ||
>> +
> client_hdl->reg_cbk.saNtfNotificationDiscardedCallback
>> != NULL) {
>> +                    /* Create a dummy msg */
>> +                    ntfsv_msg_t *msg = malloc(sizeof(ntfsv_msg_t));
>> +                    memset(msg, 0, sizeof(ntfsv_msg_t));
>> +                    msg->info.cbk_info.type = NTFSV_DUMMY_CALLBACK;
>> +                    /* Send dummy msg to client mailbox */
>> +                    if (m_NCS_IPC_SEND(&client_hdl->mbx, msg,
>> MDS_SEND_PRIORITY_HIGH) != NCSCC_RC_SUCCESS) {
>> +                            TRACE_1("m_NCS_IPC_SEND Failed to
>> client(id:%u)", client_hdl->ntfs_client_id);
>> +                            ntfa_msg_destroy(msg);
>> +                    }
>> +            }
>> +
>> +            client_hdl = client_hdl->next;
>> +    }
>> +    TRACE_LEAVE();
>> +}
>> +/****************************************************************
>> ************
>> +  Name          : ntfa_update_ntfsv_state
>> +
>> +  Description   : Update current NTF Server state by the @changedState
>> indicated
>> +              by MDS event
>> +  Arguments     : @changedState [IN]: state to be changed of NTF Server
>> +
>> +  Return Values : None
>> +
>> +  Notes         : None
>> +*****************************************************************
>> *************/
>> +void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState)
>> +{
>> +    TRACE_ENTER();
>> +    TRACE_1("Current state: %u, Changed state: %u",
>> ntfa_cb.ntfa_ntfsv_state,
>> +
>>                      changedState);
>> +
>> +    ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
>> +
>> +    switch (ntfa_cb.ntfa_ntfsv_state){
>> +    case NTFA_NTFSV_NONE:
>> +            ntfa_cb.ntfa_ntfsv_state = changedState;
>> +            break;
>> +    case NTFA_NTFSV_DOWN:
>> +            if (changedState == NTFA_NTFSV_NEW_ACTIVE ||
>> +                    changedState == NTFA_NTFSV_UP) {
>> +                    TRACE("Active NTF server has been restarted");
>> +                    ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_UP;
>> +                    ntfa_notify_handle_invalid();
>> +            } else
>> +                    TRACE("Unexpected state changes");
>> +            break;
>> +    case NTFA_NTFSV_NO_ACTIVE:
>> +            if (changedState == NTFA_NTFSV_NEW_ACTIVE) {
>> +                    TRACE("Standby NTF server becomes new Active");
>> +                    /* NTF server is functioning normally */
>> +                    ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_UP;
>> +            } else if (changedState == NTFA_NTFSV_DOWN) {
>> +                    TRACE("Active NTF Server is Down");
>> +                    ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_DOWN;
>> +                    /* Mark all client handles are invalid */
>> +                    while (client_hdl != NULL) {
>> +                            client_hdl->valid = false;
>> +                            client_hdl = client_hdl->next;
>> +                    }
>> +            }
>> +            break;
>> +    case NTFA_NTFSV_NEW_ACTIVE:
>> +            TRACE("Unknown");
>> +            break;
>> +    case NTFA_NTFSV_UP:
>> +            if (changedState == NTFA_NTFSV_NO_ACTIVE) {
>> +                    TRACE("Active NTF server temporarily unavailable");
>> +                    /* Failover/Switchover is happening
>> +                     * Any API calls result in TRY_AGAIN
>> +                     */
>> +                    ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_NO_ACTIVE;
>> +            } else
>> +                    TRACE("Unexpected state changes");
>> +            break;
>> +    default:
>> +            osafassert(false);
>> +    }
>> +
>> +    TRACE_LEAVE();
>> +}
>> +/****************************************************************
>> ************
>> +  Name          : ntfa_copy_ntf_filter_ptrs
>> +
>> +  Description   : Copy a list of filter from pSrc to pDes
>> +
>> +  Arguments     : pDes* [OUT]: list of outcome filter
>> +              pSrc* [IN]: list of input filter
>> +
>> +  Return Values : SA_AIS_OK if succeed, other values as failed
>> +
>> +  Notes         : None
>> +*****************************************************************
>> *************/
>> +SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
>> +                                                            const
>> ntfsv_filter_ptrs_t* pSrc) {
>> +    SaAisErrorT rc = SA_AIS_OK;
>> +    SaNtfNotificationFilterHeaderT *des_header;
>> +    SaNtfNotificationFilterHeaderT *src_header;
>> +    TRACE_ENTER();
>> +
>> +    if (pSrc->alarm_filter) {
>> +            pDes->alarm_filter = calloc(1,
>> sizeof(SaNtfAlarmNotificationFilterT));
>> +            des_header =
> &(pDes->alarm_filter->notificationFilterHeader);
>> +            src_header =
> &(pSrc->alarm_filter->notificationFilterHeader);
>> +            if ((rc = ntfsv_filter_header_alloc(des_header, src_header-
>>> numEventTypes,
>> +
>>              src_header->numNotificationObjects,
>> +
>>              src_header->numNotifyingObjects,
>> +
>>              src_header->numNotificationClassIds)) != SA_AIS_OK)
>> +                    goto done;
>> +            if ((rc = ntfsv_filter_alarm_alloc(pDes->alarm_filter,
>> +
>>              pSrc->alarm_filter->numProbableCauses,
>> +
>>              pSrc->alarm_filter->numPerceivedSeverities,
>> +
>>              pSrc->alarm_filter->numTrends)) != SA_AIS_OK)
>> +                    goto done;
>> +            if ((rc = ntfsv_copy_ntf_filter_alarm(pDes->alarm_filter,
>> +
>>                      pSrc->alarm_filter)) != SA_AIS_OK)
>> +                    goto done;
>> +    }
>> +
>> +    if (pSrc->sec_al_filter) {
>> +            pDes->sec_al_filter = calloc(1,
>> sizeof(SaNtfSecurityAlarmNotificationFilterT));
>> +            des_header =
> &(pDes->sec_al_filter->notificationFilterHeader);
>> +            src_header =
> &(pSrc->sec_al_filter->notificationFilterHeader);
>> +            if ((rc = ntfsv_filter_header_alloc(des_header, src_header-
>>> numEventTypes,
>> +
>>              src_header->numNotificationObjects,
>> +
>>              src_header->numNotifyingObjects,
>> +
>>              src_header->numNotificationClassIds)) != SA_AIS_OK)
>> +                    goto done;
>> +            if ((rc = ntfsv_filter_sec_alarm_alloc(pDes->sec_al_filter,
>> +
>>              pSrc->sec_al_filter->numProbableCauses,
>> +
>>              pSrc->sec_al_filter->numSeverities,
>> +
>>              pSrc->sec_al_filter->numSecurityAlarmDetectors,
>> +
>>              pSrc->sec_al_filter->numServiceUsers,
>> +
>>              pSrc->sec_al_filter->numServiceProviders)) != SA_AIS_OK)
>> +                    goto done;
>> +            if ((rc =
> ntfsv_copy_ntf_filter_sec_alarm(pDes->sec_al_filter,
>> +
>>                      pSrc->sec_al_filter)) != SA_AIS_OK)
>> +                    goto done;
>> +    }
>> +
>> +    if (pSrc->sta_ch_filter) {
>> +            pDes->sta_ch_filter = calloc(1,
>> sizeof(SaNtfStateChangeNotificationFilterT));
>> +            des_header =
> &(pDes->sta_ch_filter->notificationFilterHeader);
>> +            src_header =
> &(pSrc->sta_ch_filter->notificationFilterHeader);
>> +            if ((rc = ntfsv_filter_header_alloc(des_header, src_header-
>>> numEventTypes,
>> +
>>              src_header->numNotificationObjects,
>> +
>>              src_header->numNotifyingObjects,
>> +
>>              src_header->numNotificationClassIds)) != SA_AIS_OK)
>> +                    goto done;
>> +            if ((rc = ntfsv_filter_state_ch_alloc(pDes->sta_ch_filter,
>> +
>>              pSrc->sta_ch_filter->numSourceIndicators,
>> +
>>              pSrc->sta_ch_filter->numStateChanges)) != SA_AIS_OK)
>> +                    goto done;
>> +            if ((rc =
> ntfsv_copy_ntf_filter_state_ch(pDes->sta_ch_filter,
>> +
>>                      pSrc->sta_ch_filter)) != SA_AIS_OK)
>> +                    goto done;
>> +    }
>> +
>> +    if (pSrc->obj_cr_del_filter) {
>> +            pDes->obj_cr_del_filter = calloc(1,
>> sizeof(SaNtfObjectCreateDeleteNotificationFilterT));
>> +            des_header = &(pDes->obj_cr_del_filter-
>>> notificationFilterHeader);
>> +            src_header = &(pSrc->obj_cr_del_filter-
>>> notificationFilterHeader);
>> +            if ((rc = ntfsv_filter_header_alloc(des_header, src_header-
>>> numEventTypes,
>> +
>>              src_header->numNotificationObjects,
>> +
>>              src_header->numNotifyingObjects,
>> +
>>              src_header->numNotificationClassIds)) != SA_AIS_OK)
>> +                    goto done;
>> +            if ((rc =
> ntfsv_filter_obj_cr_del_alloc(pDes->obj_cr_del_filter,
>> +
>>              pSrc->obj_cr_del_filter->numSourceIndicators)) != SA_AIS_OK)
>> +                    goto done;
>> +            if ((rc = ntfsv_copy_ntf_filter_obj_cr_del(pDes-
>>> obj_cr_del_filter,
>> +
>>                      pSrc->obj_cr_del_filter)) != SA_AIS_OK)
>> +                    goto done;
>> +    }
>> +
>> +    if (pSrc->att_ch_filter) {
>> +            pDes->att_ch_filter = calloc(1,
>> sizeof(SaNtfAttributeChangeNotificationFilterT));
>> +            des_header =
> &(pDes->att_ch_filter->notificationFilterHeader);
>> +            src_header =
> &(pSrc->att_ch_filter->notificationFilterHeader);
>> +            if ((rc = ntfsv_filter_header_alloc(des_header, src_header-
>>> numEventTypes,
>> +
>>              src_header->numNotificationObjects,
>> +
>>              src_header->numNotifyingObjects,
>> +
>>              src_header->numNotificationClassIds)) != SA_AIS_OK)
>> +                    goto done;
>> +            if ((rc =
> ntfsv_filter_attr_change_alloc(pDes->att_ch_filter,
>> +
>>              pSrc->att_ch_filter->numSourceIndicators)) != SA_AIS_OK)
>> +                    goto done;
>> +            if ((rc = ntfsv_copy_ntf_filter_attr_ch(pDes->att_ch_filter,
>> +
>>                      pSrc->att_ch_filter)) != SA_AIS_OK)
>> +                    goto done;
>> +    }
>> +done:
>> +    TRACE_LEAVE();
>> +    return rc;
>> +}
>> +
>> +/****************************************************************
>> ************
>> +  Name          : ntfa_del_ntf_filter_ptrs
>> +
>> +  Description   : Delete the filter pointers
>> +
>> +  Arguments     : filter_ptrs* [IN/OUT]: filter pointers
>> +
>> +  Return Values : SA_AIS_OK if succeed, other values as failed
>> +
>> +  Notes         : None
>> +*****************************************************************
>> *************/
>> +SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t* filter_ptrs)
>> +{
>> +    SaAisErrorT rc = SA_AIS_OK;
>> +    if (filter_ptrs->alarm_filter) {
>> +            ntfsv_filter_alarm_free(filter_ptrs->alarm_filter, true);
>> +            free(filter_ptrs->alarm_filter);
>> +    }
>> +
>> +    if (filter_ptrs->sec_al_filter) {
>> +            ntfsv_filter_sec_alarm_free(filter_ptrs->sec_al_filter,
> true);
>> +            free(filter_ptrs->sec_al_filter);
>> +    }
>> +
>> +    if (filter_ptrs->sta_ch_filter) {
>> +            ntfsv_filter_state_ch_free(filter_ptrs->sta_ch_filter,
> true);
>> +            free(filter_ptrs->sta_ch_filter);
>> +    }
>> +
>> +    if (filter_ptrs->obj_cr_del_filter) {
>> +            ntfsv_filter_obj_cr_del_free(filter_ptrs->obj_cr_del_filter,
> true);
>> +            free(filter_ptrs->obj_cr_del_filter);
>> +    }
>> +
>> +    if (filter_ptrs->att_ch_filter) {
>> +            ntfsv_filter_attr_ch_free(filter_ptrs->att_ch_filter, true);
>> +            free(filter_ptrs->att_ch_filter);
>> +    }
>> +    return rc;
>> +}
>


------------------------------------------------------------------------------
Site24x7 APM Insight: Get Deep Visibility into Application Performance
APM + Mobile APM + RUM: Monitor 3 App instances at just $35/Month
Monitor end-to-end web transactions and take corrective actions now
Troubleshoot faster and improve end-user experience. Signup Now!
http://pubads.g.doubleclick.net/gampad/clk?id=272487151&iu=/4140
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel

Reply via email to