Hi!

See my comment inline, marked [AndersW].

regards,
Anders Widell

On 02/19/2016 07:01 AM, praveen malviya wrote:
> Please see responses with [Praveen].
>
> Thanks,
> Praveen
>
> On 18-Feb-16 9:41 PM, Lennart Lund wrote:
>> Please see responses with [Lennart]
>>
>> Thanks
>> Lennart
>>
>>> -----Original Message-----
>>> From: praveen malviya [mailto:praveen.malv...@oracle.com]
>>> Sent: den 18 februari 2016 14:02
>>> To: Minh Chau H; Lennart Lund; Vu Minh Nguyen
>>> Cc: opensaf-devel@lists.sourceforge.net
>>> Subject: Re: [PATCH 1 of 5] NTF: Add support cloud resilience for NTF Agent
>>> [#1180]
>>>
>>> Please see the responses with [Praveen]
>>>
>>> Thanks,
>>> Praveen
>>>
>>> On 12-Feb-16 8:52 PM, minh chau wrote:
>>>> Hi Praveen,
>>>>
>>>> Please find my comments inline [Minh]
>>>>
>>>> Thanks,
>>>> Minh
>>>>
>>>> On 12/02/16 22:10, praveen malviya wrote:
>>>>> Hi Minh,
>>>>>
>>>>> Please find some initial comments and questions marked with [Praveen].
>>>>>
>>>>> One question regarding the approach.
>>>>> In most of the changed APIs from saNtfInitialize() to
>>>>> saNtfNotificationReadFinalize(), clients are being recovered. Why the
>>>>> idea of recovering them is not considered in mds callback itself in
>>>>> the function call ntfa_update_ntfsv_state() when server state changes
>>>>> from DOWN to UP. It will save changes in APIs for recovering the
>>>>> clients. Clients are being marked invalid in the mds_callback itself.
>>>>>
>>>> [Minh] From memory I did try to recover clients in mds callback
>>>> (svc_evt) where indicates server UP. Though recovery needs to send
>>>> messages (reintroducing client id, read id, ...) to server, that will
>>>> end up another mds enc callback inside of first callback. That results
>>>> into fail to send *recovery* messages, there are two mds callback but
>>>> only one return. To solve this, I need to start a thread the recover
>>>> clients in background. That would cause another problem where up calls
>>>> from client could be blocked, since the recovery thread needs to block
>>>> mutual resource handles.
>>>> Another realistic reason, client can give up to send/read/receive
>>>> notification after receiving couple times of TRY_AGAIN due to server
>>>> down, and doing nothing until finalize handle. And after client stops
>>>> TRY_AGAIN loop, server can be restarted. In such cases, no needs to
>>>> recover client handles.
>>>>> Thanks,
>>>>> Praveen
>>>>>
>>>>> On 23-Dec-15 9:32 AM, Minh Hon Chau wrote:
>>>>>>     osaf/libs/agents/saf/ntfa/ntfa.h |   31 +-
>>>>>>     osaf/libs/agents/saf/ntfa/ntfa_api.c  |  672
>>>>>> +++++++++++++++++++++++++++------
>>>>>>     osaf/libs/agents/saf/ntfa/ntfa_mds.c  |    6 +-
>>>>>>     osaf/libs/agents/saf/ntfa/ntfa_util.c |  465
>>> ++++++++++++++++++++++-
>>>>>>     4 files changed, 1022 insertions(+), 152 deletions(-)
>>>>>>
>>>>>>
>>>>>> The patch contains support for cloud resilience feature
>>>>>> in NTF Agent code. Please refer README.HYDRA for content
>>>>>> of the changes
>>>>>>
>>>>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa.h
>>>>>> b/osaf/libs/agents/saf/ntfa/ntfa.h
>>>>>> --- a/osaf/libs/agents/saf/ntfa/ntfa.h
>>>>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa.h
>>>>>> @@ -91,6 +91,7 @@ typedef struct ntfa_filter_hdl_rec {
>>>>>>     typedef struct subscriberList {
>>>>>>         SaNtfHandleT subscriberListNtfHandle;
>>>>>>         SaNtfSubscriptionIdT subscriberListSubscriptionId;
>>>>>> +    ntfsv_filter_ptrs_t filters; /* remember the filters used by
>>>>>> this subscriber */
>>>>>>         struct subscriberList *prev;
>>>>>>         struct subscriberList *next;
>>>>>>     } ntfa_subscriber_list_t;
>>>>>> @@ -100,6 +101,10 @@ typedef struct ntfa_reader_hdl_rec {
>>>>>>         unsigned int reader_id;    /* handle value returned by NTFS for
>>>>>> this client */
>>>>>>         SaNtfHandleT ntfHandle;
>>>>>>         unsigned int reader_hdl;    /* READER handle from handle mgr */
>>>>>> +
>>>>>> +    ntfsv_filter_ptrs_t filters; /* remember the filters used by
>>>>>> this reader */
>>>>>> +    SaNtfSearchCriteriaT searchCriteria; /* remember the
>>>>>> searchCriteria for recovery */
>>>>>> +
>>>>>>         struct ntfa_reader_hdl_rec *next;    /* next pointer for the
>>>>>> list in ntfa_cb_t */
>>>>>>         struct ntfa_client_hdl_rec *parent_hdl;    /* Back Pointer to
>>>>>> the client instantiation */
>>>>>>     } ntfa_reader_hdl_rec_t;
>>>>>> @@ -114,24 +119,35 @@ typedef struct ntfa_client_hdl_rec {
>>>>>>         ntfa_reader_hdl_rec_t *reader_list;
>>>>>>         SYSF_MBX mbx;        /* priority q mbx b/w MDS & Library */
>>>>>>         struct ntfa_client_hdl_rec *next;    /* next pointer for the
>>>>>> list in ntfa_cb_t */
>>>>>> +    bool valid;        /* handle is valid if it's known by NTF
>>>>>> server, used for headless hydra */
>>>>>> +    SaVersionT version; /* the API version is being used by client,
>>>>>> used for recover after headless */
>>>>>>     } ntfa_client_hdl_rec_t;
>>>>>>
>>>>>>     /*
>>>>>>      * The NTFA control block is the master anchor structure for all NTFA
>>>>>>      * instantiations within a process.
>>>>>>      */
>>>>>> +typedef enum {
>>>>>> +    NTFA_NTFSV_NONE = 0,
>>>>>> +    NTFA_NTFSV_DOWN,
>>>>>> +    NTFA_NTFSV_NO_ACTIVE,
>>>>>> +    NTFA_NTFSV_NEW_ACTIVE,
>>>>>> +    NTFA_NTFSV_UP
>>>>>> +}ntfa_ntfsv_state_t;
>>>>>> +
>>>>>>     typedef struct {
>>>>>>         pthread_mutex_t cb_lock;    /* CB lock */
>>>>>>         ntfa_client_hdl_rec_t *client_list;    /* NTFA client handle
>>>>>> database */
>>>>>>         ntfa_reader_hdl_rec_t *reader_list;
>>>>>>         MDS_HDL mds_hdl;    /* MDS handle */
>>>>>>         MDS_DEST ntfs_mds_dest;    /* NTFS absolute/virtual address */
>>>>>> -    int ntfs_up;        /* Indicate that MDS subscription
>>>>>> -                 * is complete */
>>>>>> +
>>>>>>         /* NTFS NTFA sync params */
>>>>>>         int ntfs_sync_awaited;
>>>>>>         NCS_SEL_OBJ ntfs_sync_sel;
>>>>>>         SaUint32T ntf_var_data_limit;    /* max allowed
>>>>>> variableDataSize */
>>>>>> +    /* NTF Server state */
>>>>>> +    ntfa_ntfsv_state_t ntfa_ntfsv_state;
>>>>>>     } ntfa_cb_t;
>>>>>>
>>>>>>     /* ntfa_saf_api.c */
>>>>>> @@ -149,7 +165,7 @@ extern void ntfsv_ntfa_evt_free(struct n
>>>>>>
>>>>>>     /* ntfa_init.c */
>>>>>>     extern unsigned int ntfa_startup(void);
>>>>>> -extern unsigned int ntfa_shutdown(void);
>>>>>> +extern unsigned int ntfa_shutdown(bool forced);
>>>>>>
>>>>>>     /* ntfa_hdl.c */
>>>>>>     extern SaAisErrorT ntfa_hdl_cbk_dispatch(ntfa_cb_t *,
>>>>>> ntfa_client_hdl_rec_t *, SaDispatchFlagsT);
>>>>>> @@ -159,6 +175,7 @@ extern ntfa_notification_hdl_rec_t *ntfa
>>>>>>     extern ntfa_filter_hdl_rec_t
>>>>>> *ntfa_filter_hdl_rec_add(ntfa_client_hdl_rec_t **hdl_rec);
>>>>>>     extern void ntfa_hdl_list_del(ntfa_client_hdl_rec_t **);
>>>>>>     extern uint32_t ntfa_hdl_rec_del(ntfa_client_hdl_rec_t **,
>>>>>> ntfa_client_hdl_rec_t *);
>>>>>> +extern void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **,
>>>>>> ntfa_client_hdl_rec_t *);
>>>>>>     extern uint32_t
>>>>>> ntfa_notification_hdl_rec_del(ntfa_notification_hdl_rec_t **,
>>>>>> ntfa_notification_hdl_rec_t *);
>>>>>>     extern uint32_t ntfa_filter_hdl_rec_del(ntfa_filter_hdl_rec_t **,
>>>>>> ntfa_filter_hdl_rec_t *);
>>>>>>     extern bool ntfa_validate_ntfa_client_hdl(ntfa_cb_t *ntfa_cb,
>>>>>> ntfa_client_hdl_rec_t *find_hdl_rec);
>>>>>> @@ -166,11 +183,15 @@ extern bool ntfa_validate_ntfa_client_hd
>>>>>>     /* ntfa_util.c */
>>>>>>     extern ntfa_client_hdl_rec_t
>>>>>> *ntfa_find_hdl_rec_by_client_id(ntfa_cb_t *ntfa_cb, uint32_t
>>> client_id);
>>>>>>     extern void ntfa_msg_destroy(ntfsv_msg_t *msg);
>>>>>> -extern void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t
>>>>>> *instance);
>>>>>> -extern void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t
>>>>>> +extern void ntfa_notification_destructor(ntfa_notification_hdl_rec_t
>>>>>> *instance);
>>>>>> +extern void ntfa_filter_destructor(ntfa_filter_hdl_rec_t
>>>>>>                            *notificationFilterInstance);
>>>>>>     extern ntfa_reader_hdl_rec_t
>>>>>> *ntfa_reader_hdl_rec_add(ntfa_client_hdl_rec_t **hdl_rec);
>>>>>>     extern uint32_t ntfa_reader_hdl_rec_del(ntfa_reader_hdl_rec_t **,
>>>>>> ntfa_reader_hdl_rec_t *);
>>>>>>     extern void ntfa_add_to_async_cbk_msg_list(ntfsv_msg_t ** head,
>>>>>> ntfsv_msg_t * new_node);
>>>>>>     extern uint32_t ntfa_ntfs_msg_proc(ntfa_cb_t *cb, ntfsv_msg_t
>>>>>> *ntfsv_msg, MDS_SEND_PRIORITY_TYPE prio);
>>>>>> +extern void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t
>>> changedState);
>>>>>> +extern SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t*
>>> pDes,
>>>>>> +                                const ntfsv_filter_ptrs_t* pSrc);
>>>>>> +extern SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t*
>>>>>> filter_ptrs);
>>>>>>     #endif   /* !NTFA_H */
>>>>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>>>>> b/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>>>>> --- a/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>>>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>>>>> @@ -34,11 +34,42 @@
>>>>>>     /* The main controle block */
>>>>>>     ntfa_cb_t ntfa_cb = {
>>>>>>         .cb_lock = PTHREAD_MUTEX_INITIALIZER,
>>>>>> +    .ntfa_ntfsv_state = NTFA_NTFSV_NONE,
>>>>>>     };
>>>>>>
>>>>>>     /* list of subscriptions for this process */
>>>>>>     ntfa_subscriber_list_t *subscriberNoList = NULL;
>>>>>>
>>>>>> +/*
>>>>>> + * @Brief: Determine the common "availability" of API, which depends
>>>>>> on the
>>>>>> + *         NTF Server state
>>>>>> + * @Param: None
>>>>>> + * @Return: OK - If Server state is UP
>>>>>> + *          TRY_AGAIN - If Server state is No Active (temporary no
>>>>>> active)
>>>>>> + *          BAD_HANDLE - If Server state is completely Down (no
>>>>>> active at all).
>>>>>> + *                       This error code will be returned to client
>>>>>> calling API
>>>>>> + */
>>>>>> +static SaAisErrorT checkNtfServerState()
>>>>>> +{
>>>>>> +    SaAisErrorT rc = SA_AIS_ERR_NOT_SUPPORTED;
>>>>>> +    /* Check NTF server availability */
>>>>>> +    if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_UP) {
>>>>>> +        rc = SA_AIS_OK;
>>>>>> +    } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_DOWN) {
>>>>>> +        TRACE("NTFS server is down");
>>>>>> +        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>> [Praveen] 1. Function header says BAD_HANDLE for "down" but here
>>>>> TRY_AGAIN is returned.
>>>> [Minh]: I got similar comments from Lennart, I will correct the function
>>>> description
>>>>> 2. If server state is down, it means both ntfs are not up which means
>>>>> headless cluster. But in case of normal cluster also application will
>>>>> get TRY_AGAIN. Are you relying on the              fact that in normal
>>>>> cluster case payloads will be rebooted? if answer is yes then what if
>>>>> reboot is delayed by changing the opensaf_reboot script which is
>>>>> customizable?
>>>> [Minh]: I don't get your question. No impact if payloads reboot
>>> [Praveen] Actually I want draw attention to two problems here:
>>> 1) Normal cluster case: Both controllers are up in normal cluster
>>> (attribute commented in immd.conf) and some ntf application is running
>>> on the payload. Now if opensaf is stopped on the payload, application
>>> will still get TRY_AGIAN. Now if a fresh controller is started and
>>> payload is not started, the application API will be successful. But this
>>> application should not be able to get a success for any API since its
>>> payload  is down.
>>> 2) Headless cluster case: Same case when with headless cluster
>>> configured (attribute un-commented in immd.conf).
>>>
>>> I think this may be the generic problem with all the service with two
>>> tier architecture and a fix is needed. There should be a mechanism
>>> though which agent should detect opensaf shutdown on local node and
>>> clear all clients data.
>>>
>>>
>> [Lennart] I cannot see that there is a problem if an OpenSAF service that 
>> has no node director (ntf and log) can recover and continue to work
>> after a headless situation. For this service there is no OpenSAF running on 
>> a payload node.
>> For the "normal" case (attribute  commented in immd.conf) the current 
>> behavior shall apply which is to invalidate all clients if no server is 
>> available (SC-nodes down). There should be no impact at all in any mode on 
>> the ntf and log service if OpenSAF is stopped on the payload node.
> As I mentioned this is not the case when controller(s) are down, only
> bring down the payload when application is running on it. In this case
> NTFS will not get agent down and it will not clear this client. Also
> agent only MDS subscribes for NTFS so it will also not detect opensaf
> down of local node. So the application will continue, in my case Send()
> API was able to send notification successfully.
> Anyways I have verified, this is the problem without any headless
> patches also.
[AndersW] I think this problem is caused by missing / incomplete 
integration with CLM. If you stop OpenSAF on a payload node, the node 
will no longer be a member of the cluster. NTF ought to return 
SA_AIS_ERR_UNAVAILABLE to client requests from non-member nodes 
according to the A.03.01 spec (see Section 3.12.1 in the A.03.01 spec). 
The OpenSAF implementation is still on the A.01.01 version, though. 
There is a separate enhancement ticket to implement full CLM integration 
in NTF: ticket [#1639].
>
> Thanks
> Praveen
>>>>>> +    } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NO_ACTIVE) {
>>>>>> +        TRACE("NTFS server is unavailable");
>>>>>> +        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> +    } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NONE) {
>>>>>> +        TRACE("No NTF server is detected, or API is called during
>>>>>> headless");
>>>>>> +        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> +    } else
>>>>>> +        TRACE("Not supported API call under NTF Server state (%u)",
>>>>>> +                ntfa_cb.ntfa_ntfsv_state);
>>>>>> +    return rc;
>>>>>> +}
>>>>>> +
>>>>>>     static SaAisErrorT checkNtfValueTypeRange(SaNtfValueTypeT type)
>>>>>>     {
>>>>>>         return (type < SA_NTF_VALUE_UINT8 || type >
>>>>>> SA_NTF_VALUE_ARRAY)? SA_AIS_ERR_INVALID_PARAM
>>>>>> @@ -871,7 +902,266 @@ SaAisErrorT getFilters(const SaNtfNotifi
>>>>>>     }
>>>>>>
>>>>>>     /* end help functions */
>>>>>> -
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : reinitializeClient
>>>>>> +
>>>>>> +  Description   : This routine sends initialize messages to NTF
>>>>>> Server in
>>>>>> +          order to obtain new client id
>>>>>> +
>>>>>> +  Arguments     : client_hdl* [IN]: client handle
>>>>>> +
>>>>>> +  Return Values : SA_AIS_OK if success, others are failure
>>>>>> +
>>>>>> +  Notes         : None
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +SaAisErrorT reinitializeClient(ntfa_client_hdl_rec_t* client_hdl) {
>>>>>> +    uint32_t mds_rc;
>>>>>> +    SaAisErrorT rc = SA_AIS_OK;
>>>>>> +    ntfsv_msg_t i_msg, *o_msg = NULL;
>>>>>> +
>>>>>> +    TRACE_ENTER();
>>>>>> +
>>>>>> +    memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>>>>>> +    i_msg.type = NTFSV_NTFA_API_MSG;
>>>>>> +    i_msg.info.api_info.type = NTFSV_INITIALIZE_REQ;
>>>>>> +    i_msg.info.api_info.param.init.version = client_hdl->version;
>>>>>> +
>>>>>> +    mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>>>>>> NTFS_WAIT_TIME);
>>>>>> +    switch (mds_rc) {
>>>>>> +    case NCSCC_RC_SUCCESS:
>>>>>> +        break;
>>>>>> +    case NCSCC_RC_REQ_TIMOUT:
>>>>>> +        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> +        goto done;
>>>>>> +    default:
>>>>>> +        rc = SA_AIS_ERR_BAD_HANDLE;
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    osafassert(o_msg != NULL);
>>>>>> +    /* Check response msg: type, rc */
>>>>>> +    if (o_msg->info.api_resp_info.type != NTFSV_INITIALIZE_RSP) {
>>>>>> +        TRACE("info.api_resp_info.type:%u",
>>>>>> o_msg->info.api_resp_info.type);
>>>>>> +        rc = SA_AIS_ERR_LIBRARY;
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +    if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>>>>>> +        TRACE("info.api_resp_info.rc:%u",
>>>>>> o_msg->info.api_resp_info.rc);
>>>>>> +        rc = SA_AIS_ERR_BAD_HANDLE;
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    client_hdl->ntfs_client_id =
>>>>>> o_msg->info.api_resp_info.param.init_rsp.client_id;
>>>>>> +    TRACE("Successfully recover client_id, new client_id:%u",
>>>>>> +            client_hdl->ntfs_client_id);
>>>>>> +
>>>>>> +done:
>>>>>> +    if (o_msg)
>>>>>> +        ntfa_msg_destroy(o_msg);
>>>>>> +    TRACE_LEAVE();
>>>>>> +    return rc;
>>>>>> +}
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : recoverReader
>>>>>> +
>>>>>> +  Description   : This routine sends read_initialize messages to NTF
>>>>>> Server in
>>>>>> +          order to reintroduce this reader to NTF Server
>>>>>> +
>>>>>> +  Arguments     : client_hdl* [IN]: client handle
>>>>>> +          reader_hdl* [IN]: reader handle
>>>>>> +
>>>>>> +  Return Values : SA_AIS_OK if success, others are failure
>>>>>> +
>>>>>> +  Notes         : None
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +SaAisErrorT recoverReader(ntfa_client_hdl_rec_t* client_hdl,
>>>>>> ntfa_reader_hdl_rec_t* reader_hdl) {
>>>>>> +    uint32_t mds_rc;
>>>>>> +    SaAisErrorT rc = SA_AIS_OK;
>>>>>> +    ntfsv_msg_t i_msg, *o_msg = NULL;
>>>>>> +    ntfsv_reader_init_req_2_t *send_param;
>>>>>> +
>>>>>> +    TRACE_ENTER();
>>>>>> +
>>>>>> +    /* Make sure the filters have not been deleted */
>>>>>> +    if (reader_hdl->filters.alarm_filter == NULL &&
>>>>>> +        reader_hdl->filters.att_ch_filter == NULL &&
>>>>>> +        reader_hdl->filters.obj_cr_del_filter == NULL &&
>>>>>> +        reader_hdl->filters.sec_al_filter == NULL &&
>>>>>> +        reader_hdl->filters.sta_ch_filter == NULL)
>>>>>> +        return SA_AIS_ERR_BAD_HANDLE;
>>>>>> +
>>>>>> +    memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>>>>>> +    i_msg.type = NTFSV_NTFA_API_MSG;
>>>>>> +    i_msg.info.api_info.type = NTFSV_READER_INITIALIZE_REQ_2;
>>>>>> +    send_param = &i_msg.info.api_info.param.reader_init_2;
>>>>>> +    send_param->head.client_id = client_hdl->ntfs_client_id;
>>>>>> +    send_param->head.searchCriteria = reader_hdl->searchCriteria;
>>>>>> +    send_param->f_rec = reader_hdl->filters;
>>>>>> +
>>>>>> +    mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>>>>>> NTFS_WAIT_TIME);
>>>>>> +
>>>>>> +    switch (mds_rc) {
>>>>>> +    case NCSCC_RC_SUCCESS:
>>>>>> +        break;
>>>>>> +    case NCSCC_RC_REQ_TIMOUT:
>>>>>> +        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> +        goto done;
>>>>>> +    default:
>>>>>> +        rc = SA_AIS_ERR_BAD_HANDLE;
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    osafassert(o_msg != NULL);
>>>>>> +    if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>>>>>> +        TRACE("o_msg->info.api_resp_info.rc:%u",
>>>>>> o_msg->info.api_resp_info.rc);
>>>>>> +        rc = SA_AIS_ERR_BAD_HANDLE;
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    if (o_msg->info.api_resp_info.type !=
>>>>>> NTFSV_READER_INITIALIZE_RSP) {
>>>>>> +        TRACE("msg type (%d) failed",
>>>>>> (int)o_msg->info.api_resp_info.type);
>>>>>> +        rc = SA_AIS_ERR_LIBRARY;
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +    /* Update reader_id since it may be changed */
>>>>>> +    reader_hdl->reader_id =
>>>>>> o_msg->info.api_resp_info.param.reader_init_rsp.readerId;
>>>>>> +
>>>>>> +    TRACE("Recover reader successfully");
>>>>>> +done:
>>>>>> +    if (o_msg)
>>>>>> +        ntfa_msg_destroy(o_msg);
>>>>>> +    TRACE_LEAVE();
>>>>>> +    return rc;
>>>>>> +}
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : recoverSubscriber
>>>>>> +
>>>>>> +  Description   : This routine sends subscribe messages to NTF
>>>>>> Server in
>>>>>> +          order to reintroduce this subscriber to NTF Server
>>>>>> +
>>>>>> +  Arguments     : client_hdl* [IN]: client handle
>>>>>> +          subscriber_hdl* [IN]: subscriber handle
>>>>>> +
>>>>>> +  Return Values : SA_AIS_OK if success, others are failure
>>>>>> +
>>>>>> +  Notes         : None
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +SaAisErrorT recoverSubscriber(ntfa_client_hdl_rec_t* client_hdl,
>>>>>> +                                    ntfa_subscriber_list_t*
>>>>>> subscriber_hdl) {
>>>>>> +    uint32_t mds_rc;
>>>>>> +    SaAisErrorT rc = SA_AIS_OK;
>>>>>> +    ntfsv_msg_t i_msg, *o_msg = NULL;
>>>>>> +    ntfsv_subscribe_req_t *send_param;
>>>>>> +
>>>>>> +    TRACE_ENTER();
>>>>>> +
>>>>>> +    /* Make sure the filters have not been deleted */
>>>>>> +    if (subscriber_hdl->filters.alarm_filter == NULL &&
>>>>>> +        subscriber_hdl->filters.att_ch_filter == NULL &&
>>>>>> +        subscriber_hdl->filters.obj_cr_del_filter == NULL &&
>>>>>> +        subscriber_hdl->filters.sec_al_filter == NULL &&
>>>>>> +        subscriber_hdl->filters.sta_ch_filter == NULL)
>>>>>> +        return SA_AIS_ERR_BAD_HANDLE;
>>>>>> +
>>>>>> +    memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>>>>>> +    i_msg.type = NTFSV_NTFA_API_MSG;
>>>>>> +    i_msg.info.api_info.type = NTFSV_SUBSCRIBE_REQ;
>>>>>> +    send_param = &i_msg.info.api_info.param.subscribe;
>>>>>> +
>>>>>> +    send_param->client_id = client_hdl->ntfs_client_id;
>>>>>> +    send_param->subscriptionId =
>>>>>> subscriber_hdl->subscriberListSubscriptionId;
>>>>>> +    send_param->f_rec = subscriber_hdl->filters;
>>>>>> +
>>>>>> +    mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>>>>>> NTFS_WAIT_TIME);
>>>>>> +
>>>>>> +    switch (mds_rc) {
>>>>>> +    case NCSCC_RC_SUCCESS:
>>>>>> +        break;
>>>>>> +    case NCSCC_RC_REQ_TIMOUT:
>>>>>> +        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> +        goto done;
>>>>>> +    default:
>>>>>> +        rc = SA_AIS_ERR_BAD_HANDLE;
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    osafassert(o_msg != NULL);
>>>>>> +
>>>>>> +    if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>>>>>> +        TRACE("o_msg->info.api_resp_info.rc:%u",
>>>>>> o_msg->info.api_resp_info.rc);
>>>>>> +        rc = SA_AIS_ERR_BAD_HANDLE;
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    if (o_msg->info.api_resp_info.type != NTFSV_SUBSCRIBE_RSP) {
>>>>>> +        TRACE("msg type (%d) failed",
>>>>>> (int)o_msg->info.api_resp_info.type);
>>>>>> +        rc = SA_AIS_ERR_LIBRARY;
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +    TRACE("Recover subscriber successfully");
>>>>>> +done:
>>>>>> +    if (o_msg)
>>>>>> +        ntfa_msg_destroy(o_msg);
>>>>>> +
>>>>>> +    TRACE_LEAVE();
>>>>>> +    return rc;
>>>>>> +}
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : recoverClient
>>>>>> +
>>>>>> +  Description   : In NTF-A.03.01, section 2.1.2. A client can be
>>>>>> producer,
>>>>>> +          subscriber, and reader at the same time.
>>>>>> +          This routine will recovery:
>>>>>> +          (1) client_id
>>>>>> +          (2) Then it will continue the recovery if this client also
>>>>>> +          has instance of subscriber or reader.
>>>>>> +  Arguments     : client_hdl* [IN]: client handle
>>>>>> +
>>>>>> +  Return Values : SA_AIS_OK if success, others are failure
>>>>>> +
>>>>>> +  Notes         : None
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +SaAisErrorT recoverClient(ntfa_client_hdl_rec_t *client_hdl) {
>>>>>> +
>>>>>> +    TRACE_ENTER();
>>>>>> +    SaAisErrorT rc = SA_AIS_OK;
>>>>>> +
>>>>>> +    osafassert(client_hdl);
>>>>>> +
>>>>>> +    if ((rc = reinitializeClient(client_hdl)) == SA_AIS_OK) {
>>>>>> +        /* Restore reader */
>>>>>> +        ntfa_reader_hdl_rec_t* reader_hdl = client_hdl->reader_list;
>>>>>> +        while (reader_hdl != NULL && rc == SA_AIS_OK) {
>>>>>> +            rc = recoverReader(client_hdl, reader_hdl);
>>>>>> +            reader_hdl = reader_hdl->next;
>>>>>> +        }
>>>>>> +        if (rc != SA_AIS_OK) {
>>>>>> +            TRACE("Failed to restore reader (readerId:%d)",
>>>>>> +                    reader_hdl->reader_id);
>>>>>> +            goto done;
>>>>>> +        }
>>>>>> +        /* Restore subscriber */
>>>>>> +        ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
>>>>>> +        while (subscriber_hdl != NULL && rc == SA_AIS_OK) {
>>>>>> +            if (client_hdl->local_hdl ==
>>>>>> subscriber_hdl->subscriberListNtfHandle)
>>>>>> +                rc = recoverSubscriber(client_hdl, subscriber_hdl);
>>>>>> +            subscriber_hdl = subscriber_hdl->next;
>>>>>> +        }
>>>>>> +        if (rc != SA_AIS_OK) {
>>>>>> +            TRACE("Failed to restore subscriber (subscriptionId:%d)",
>>>>>> + subscriber_hdl->subscriberListSubscriptionId);
>>>>>> +            goto done;
>>>>>> +        }
>>>>>> +        client_hdl->valid = true;
>>>>>> +    } else {
>>>>>> +        TRACE("Failed to restore client (id:%d)",
>>>>>> client_hdl->ntfs_client_id);
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +done:
>>>>>> +    TRACE_LEAVE();
>>>>>> +    return rc;
>>>>>> +}
>>>>>>
>>> /**********************************************************
>>> *****************
>>>>>>      * 8.4.1
>>>>>>      *
>>>>>> @@ -923,15 +1213,14 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>>>>>             version->releaseCode = NTF_RELEASE_CODE;
>>>>>>             version->majorVersion = NTF_MAJOR_VERSION;
>>>>>>             version->minorVersion = NTF_MINOR_VERSION;
>>>>>> -        ntfa_shutdown();
>>>>>> +        ntfa_shutdown(false);
>>>>>>             rc = SA_AIS_ERR_VERSION;
>>>>>>             goto done;
>>>>>>         }
>>>>>>
>>>>>> -    if (!ntfa_cb.ntfs_up) {
>>>>>> -        ntfa_shutdown();
>>>>>> -        TRACE("NTFS server is down");
>>>>>> -        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> +    /* Check NTF server availability */
>>>>>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>>>>>> +        ntfa_shutdown(false);
>>>>>>             goto done;
>>>>>>         }
>>>>>>
>>>>>> @@ -977,7 +1266,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>>>>>             rc = SA_AIS_ERR_NO_MEMORY;
>>>>>>             goto err;
>>>>>>         }
>>>>>> -
>>>>>> +    ntfa_hdl_rec->version = *version;
>>>>>>         /* pass the handle value to the appl */
>>>>>>         if (SA_AIS_OK == rc)
>>>>>>             *ntfHandle = ntfa_hdl_rec->local_hdl;
>>>>>> @@ -990,7 +1279,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>>>>>
>>>>>>         if (rc != SA_AIS_OK) {
>>>>>>             TRACE_2("NTFA INIT FAILED\n");
>>>>>> -        ntfa_shutdown();
>>>>>> +        ntfa_shutdown(false);
>>>>>>         }
>>>>>>
>>>>>>      done:
>>>>>> @@ -1101,6 +1390,28 @@ SaAisErrorT saNtfDispatch(SaNtfHandleT n
>>>>>>             goto done;
>>>>>>         }
>>>>>>
>>>>>> +    /* Need to check NTF server availability here
>>>>>> +     * Because if the notificationCallback just comes after MDS_DOWN
>>>>>> +     * it will go to recovery then result in failure due to no director
>>>>>> +     */
>>>>>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>>>>>> +        ncshm_give_hdl(ntfHandle);
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    if (!hdl_rec->valid) {
>>>>>> +        /* recovery */
>>>>>> +        if ((rc = recoverClient(hdl_rec)) != SA_AIS_OK) {
>>>>>> +            if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>>>>> +                ncshm_give_hdl(ntfHandle);
>>>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, hdl_rec);
>>>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>>> +                ntfa_shutdown(false);
>>>>>> +                goto done;
>>>>>> +            }
>>>>>> +        }
>>>>>> +    }
>>>>>>         if ((rc = ntfa_hdl_cbk_dispatch(&ntfa_cb, hdl_rec,
>>>>>> dispatchFlags)) != SA_AIS_OK)
>>>>>>             TRACE("NTFA_DISPATCH_FAILURE");
>>>>>>
>>>>>> @@ -1145,6 +1456,13 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>>>>>
>>>>>>         TRACE_ENTER();
>>>>>>
>>>>>> +    /* Check NTF server availability */
>>>>>> +    if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NO_ACTIVE) {
>>>>>> +        TRACE("NTFS server is temporarily unavailable");
>>>>>> +        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +
>>>>>>         /* retrieve hdl rec */
>>>>>>         hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA, ntfHandle);
>>>>>>         if (hdl_rec == NULL) {
>>>>>> @@ -1152,43 +1470,37 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>>>>>             rc = SA_AIS_ERR_BAD_HANDLE;
>>>>>>             goto done;
>>>>>>         }
>>>>>> -
>>>>>> -    /* Check Whether NTFS is up or not */
>>>>>> -    if (!ntfa_cb.ntfs_up) {
>>>>>> -        TRACE("NTFS down");
>>>>>> -        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> -        goto done_give_hdl;
>>>>>> +    if (hdl_rec->valid) {
>>>>>> +        /** populate & send the finalize message
>>>>>> +         ** and make sure the finalize from the server
>>>>>> +         ** end returned before deleting the local records.
>>>>>> +         **/
>>>>>> +        memset(&msg, 0, sizeof(ntfsv_msg_t));
>>>>>> +        msg.type = NTFSV_NTFA_API_MSG;
>>>>>> +        msg.info.api_info.type = NTFSV_FINALIZE_REQ;
>>>>>> +        msg.info.api_info.param.finalize.client_id =
>>>>>> hdl_rec->ntfs_client_id;
>>>>>> +
>>>>>> +        mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>>>>>> NTFS_WAIT_TIME);
>>>>>> +        switch (mds_rc) {
>>>>>> +        case NCSCC_RC_SUCCESS:
>>>>>> +            break;
>>>>>> +        case NCSCC_RC_REQ_TIMOUT:
>>>>>> +            rc = SA_AIS_ERR_TIMEOUT;
>>>>>> +            TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>>>>> +            goto done_give_hdl;
>>>>>> +        default:
>>>>>> +            TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>>>>> +            rc = SA_AIS_ERR_NO_RESOURCES;
>>>>>> +            goto done_give_hdl;
>>>>>> +        }
>>>>>> +
>>>>>> +        if (o_msg != NULL) {
>>>>>> +            rc = o_msg->info.api_resp_info.rc;
>>>>>> +            ntfa_msg_destroy(o_msg);
>>>>>> +        } else
>>>>>> +            rc = SA_AIS_ERR_NO_RESOURCES;
>>>>>>         }
>>>>>>
>>>>>> -    /** populate & send the finalize message
>>>>>> -     ** and make sure the finalize from the server
>>>>>> -     ** end returned before deleting the local records.
>>>>>> -     **/
>>>>>> -    memset(&msg, 0, sizeof(ntfsv_msg_t));
>>>>>> -    msg.type = NTFSV_NTFA_API_MSG;
>>>>>> -    msg.info.api_info.type = NTFSV_FINALIZE_REQ;
>>>>>> -    msg.info.api_info.param.finalize.client_id =
>>>>>> hdl_rec->ntfs_client_id;
>>>>>> -
>>>>>> -    mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>>>>>> NTFS_WAIT_TIME);
>>>>>> -    switch (mds_rc) {
>>>>>> -    case NCSCC_RC_SUCCESS:
>>>>>> -        break;
>>>>>> -    case NCSCC_RC_REQ_TIMOUT:
>>>>>> -        rc = SA_AIS_ERR_TIMEOUT;
>>>>>> -        TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>>>>> -        goto done_give_hdl;
>>>>>> -    default:
>>>>>> -        TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>>>>> -        rc = SA_AIS_ERR_NO_RESOURCES;
>>>>>> -        goto done_give_hdl;
>>>>>> -    }
>>>>>> -
>>>>>> -    if (o_msg != NULL) {
>>>>>> -        rc = o_msg->info.api_resp_info.rc;
>>>>>> -        ntfa_msg_destroy(o_msg);
>>>>>> -    } else
>>>>>> -        rc = SA_AIS_ERR_NO_RESOURCES;
>>>>>> -
>>>>>>         if (rc == SA_AIS_OK) {
>>>>>>         /** delete the hdl rec
>>>>>>              ** including all resources allocated by client if MDS send 
>>>>>> is
>>>>>> @@ -1205,7 +1517,7 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>>>>>         ncshm_give_hdl(ntfHandle);
>>>>>>
>>>>>>         if (rc == SA_AIS_OK) {
>>>>>> -        rc = ntfa_shutdown();
>>>>>> +        rc = ntfa_shutdown(false);
>>>>>>             if (rc != NCSCC_RC_SUCCESS)
>>>>>>                 TRACE_1("ntfa_shutdown failed");
>>>>>>         }
>>>>>> @@ -1349,7 +1661,7 @@ SaAisErrorT saNtfNotificationFree(SaNtfN
>>>>>>         }
>>>>>>
>>>>>>         /* free the resources allocated by
>>>>>> saNtf<ntfType>NotificationAllocate */
>>>>>> -    ntfa_hdl_rec_destructor(notification_hdl_rec);
>>>>>> +    ntfa_notification_destructor(notification_hdl_rec);
>>>>>>
>>>>>>         /** Delete the resources related to the notificationHandle &
>>>>>>          *  remove reference in the client.
>>>>>> @@ -1419,6 +1731,24 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>>>>>>         }
>>>>>>         osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>>>
>>>>>> +    /* Check NTF server availability */
>>>>>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>>>>> +        goto done_give_hdls;
>>>>>> +
>>>>>> +    /* Recover if this is invalid handle */
>>>>>> +    if (!client_rec->valid) {
>>>>>> +        if ((rc = recoverClient(client_rec)) != SA_AIS_OK) {
>>>>>> +            ncshm_give_hdl(client_handle);
>>>>>> +            ncshm_give_hdl(notificationHandle);
>>>>>> +            if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_rec);
>>>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>>> +                ntfa_shutdown(false);
>>>>>> +            }
>>>>>> +            goto err_free;
>>>>>> +        }
>>>>>> +    }
>>>>>>         /**
>>>>>>          ** Populate a sync MDS message
>>>>>>          **/
>>>>>> @@ -1481,13 +1811,6 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>>>>>>             goto done_give_hdls;
>>>>>>         }
>>>>>>
>>>>>> -    /* Check whether NTFS is up or not */
>>>>>> -    if (!ntfa_cb.ntfs_up) {
>>>>>> -        TRACE("NTFS down");
>>>>>> -        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> -        goto done_give_hdls;
>>>>>> -    }
>>>>>> -
>>>>>>         /* Send a sync MDS message to obtain a notification id */
>>>>>>         mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>>> timeout);
>>>>>>         switch (mds_rc) {
>>>>>> @@ -1596,7 +1919,8 @@ static  SaNtfHandleT ntfHandleGet(SaNtfS
>>>>>>         return ntfHandle;
>>>>>>     }
>>>>>>
>>>>>> -static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
>>>>>> SaNtfSubscriptionIdT subscriptionId)
>>>>>> +static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
>>>>>> SaNtfSubscriptionIdT subscriptionId,
>>>>>> +                                        ntfsv_filter_ptrs_t filters)
>>>>>>     {
>>>>>>         SaAisErrorT rc = SA_AIS_OK;
>>>>>>         ntfa_subscriber_list_t* ntfSubscriberList;
>>>>>> @@ -1610,7 +1934,9 @@ static SaAisErrorT subscriptionListAdd(S
>>>>>>         /* Add ntfHandle and subscriptionId into list */
>>>>>>         ntfSubscriberList->subscriberListNtfHandle = ntfHandle;
>>>>>>         ntfSubscriberList->subscriberListSubscriptionId = subscriptionId;
>>>>>> -
>>>>>> +    memset(&ntfSubscriberList->filters, 0,
>>>>>> sizeof(ntfsv_filter_ptrs_t));
>>>>>> + ntfa_copy_ntf_filter_ptrs(&ntfSubscriberList->filters, &filters);
>>>>>> +
>>>>>>         osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>>>>         if (NULL == subscriberNoList) {
>>>>>>             subscriberNoList = ntfSubscriberList;
>>>>>> @@ -1648,6 +1974,7 @@ static void subscriberListItemRemove(SaN
>>>>>>                 subscriberNoList = NULL;
>>>>>>         }
>>>>>>         TRACE_1("REMOVE: listPtr->SubscriptionId %d",
>>>>>> listPtr->subscriberListSubscriptionId);
>>>>>> +    ntfa_del_ntf_filter_ptrs(&listPtr->filters);
>>>>>>         free(listPtr);
>>>>>>         osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>>>     }
>>>>>> @@ -1666,20 +1993,33 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>>>>>         ntfsv_msg_t msg, *o_msg = NULL;
>>>>>>         ntfsv_subscribe_req_t *send_param;
>>>>>>         uint32_t timeout = NTFS_WAIT_TIME;
>>>>>> -
>>>>>> +    bool recovery_failed = false;
>>>>>>         TRACE_ENTER();
>>>>>> +
>>>>>>         rc = getFilters(notificationFilterHandles, &filters,
>>>>>> &ntfHandle, &client_hdl_rec);
>>>>>>         if (rc != SA_AIS_OK) {
>>>>>>             TRACE("getFilters failed");
>>>>>>             goto done;
>>>>>>         }
>>>>>>
>>>>>> +    /* Check NTF server availability */
>>>>>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>>>>> +        goto done;
>>>>>> +
>>>>>> +    /* recovery */
>>>>>> +    if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
>>>>>> +        if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>>>>> +            recovery_failed = true;
>>>>>> +            goto done;
>>>>>> +        }
>>>>>> +    }
>>>>>> +
>>>>>>         tmpHandle = ntfHandleGet(subscriptionId);
>>>>>>         if (tmpHandle != 0) {
>>>>>>             rc = SA_AIS_ERR_EXIST;
>>>>>>             goto done;
>>>>>>         }
>>>>>> -    rc = subscriptionListAdd(ntfHandle, subscriptionId);
>>>>>> +    rc = subscriptionListAdd(ntfHandle, subscriptionId, filters);
>>>>>>         if (rc != SA_AIS_OK) {
>>>>>>             goto done;
>>>>>>         }
>>>>>> @@ -1693,29 +2033,24 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>>>>>         send_param->client_id = client_hdl_rec->ntfs_client_id;
>>>>>>         send_param->subscriptionId = subscriptionId;
>>>>>>         send_param->f_rec = filters;
>>>>>> -    /* Check whether NTFS is up or not */
>>>>>> -    if (ntfa_cb.ntfs_up) {
>>>>>> -        uint32_t rv;
>>>>>> -        /* Send a sync MDS message to obtain a log stream id */
>>>>>> -        rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>>> timeout);
>>>>>> -        if (rv == NCSCC_RC_SUCCESS) {
>>>>>> -            osafassert(o_msg != NULL);
>>>>>> -            if (SA_AIS_OK == o_msg->info.api_resp_info.rc) {
>>>>>> -                TRACE_1("subscriptionId from server %u",
>>>>>> - o_msg->info.api_resp_info.param.subscribe_rsp.subscriptionId);
>>>>>> -            } else {
>>>>>> -                rc = o_msg->info.api_resp_info.rc;
>>>>>> -                TRACE("Bad return status!!! rc = %d", rc);
>>>>>> -            }
>>>>>> +
>>>>>> +    uint32_t rv;
>>>>>> +    /* Send a sync MDS message to obtain a log stream id */
>>>>>> +    rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>>>>> +    if (rv == NCSCC_RC_SUCCESS) {
>>>>>> +        osafassert(o_msg != NULL);
>>>>>> +        if (SA_AIS_OK == o_msg->info.api_resp_info.rc) {
>>>>>> +            TRACE_1("subscriptionId from server %u",
>>>>>> + o_msg->info.api_resp_info.param.subscribe_rsp.subscriptionId);
>>>>>>             } else {
>>>>>> -            if(rv == NCSCC_RC_INVALID_INPUT)
>>>>>> -                rc = SA_AIS_ERR_INVALID_PARAM;
>>>>>> -            else
>>>>>> -                rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> +            rc = o_msg->info.api_resp_info.rc;
>>>>>> +            TRACE("Bad return status!!! rc = %d", rc);
>>>>>>             }
>>>>>>         } else {
>>>>>> -        TRACE_1("NTFS down");
>>>>>> -        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> +        if(rv == NCSCC_RC_INVALID_INPUT)
>>>>>> +            rc = SA_AIS_ERR_INVALID_PARAM;
>>>>>> +        else
>>>>>> +            rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>>         }
>>>>>>
>>>>>>         if (rc != SA_AIS_OK) {
>>>>>> @@ -1724,6 +2059,7 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>>>>>         if (o_msg)
>>>>>>             ntfa_msg_destroy(o_msg);
>>>>>>         done:
>>>>>> +
>>>>>>         ncshm_give_hdl(ntfHandle);
>>>>>>         if (notificationFilterHandles) {
>>>>>>             if (notificationFilterHandles->attributeChangeFilterHandle)
>>>>>> @@ -1737,6 +2073,13 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>>>>>             if (notificationFilterHandles->alarmFilterHandle)
>>>>>> ncshm_give_hdl(notificationFilterHandles->alarmFilterHandle);
>>>>>>         }
>>>>>> +    if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
>>>>>> +        osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>>>> +        ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>>>>> +        osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>>> +        ntfa_shutdown(false);
>>>>>> +    }
>>>>>> +
>>>>>>         TRACE_LEAVE();
>>>>>>         return rc;
>>>>>>     }
>>>>>> @@ -1995,6 +2338,7 @@ SaAisErrorT saNtfSecurityAlarmNotificati
>>>>>>             rc = SA_AIS_ERR_INVALID_PARAM;
>>>>>>             goto done;
>>>>>>         }
>>>>>> +
>>>>>>         /* retrieve hdl rec */
>>>>>>         hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA, ntfHandle);
>>>>>>         if (hdl_rec == NULL) {
>>>>>> @@ -2058,6 +2402,7 @@ SaAisErrorT saNtfPtrValAllocate(SaNtfNot
>>>>>>             rc = SA_AIS_ERR_INVALID_PARAM;
>>>>>>             goto done;
>>>>>>         }
>>>>>> +
>>>>>>         notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>>>>>> notificationHandle);
>>>>>>         if (notification_hdl_rec == NULL) {
>>>>>>             TRACE("ncshm_take_hdl notificationHandle failed");
>>>>>> @@ -2098,6 +2443,7 @@ SaAisErrorT saNtfArrayValAllocate(SaNtfN
>>>>>>             rc = SA_AIS_ERR_INVALID_PARAM;
>>>>>>             goto done;
>>>>>>         }
>>>>>> +
>>>>>>         notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>>>>>> notificationHandle);
>>>>>>         if (notification_hdl_rec == NULL) {
>>>>>>             TRACE("ncshm_take_hdl notificationHandle failed");
>>>>>> @@ -2228,6 +2574,7 @@ SaAisErrorT saNtfArrayValGet(SaNtfNotifi
>>>>>>             rc = SA_AIS_ERR_INVALID_PARAM;
>>>>>>             goto done;
>>>>>>         }
>>>>>> +
>>>>>>         notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>>>>>> notificationHandle);
>>>>>>         if (notification_hdl_rec == NULL) {
>>>>>>             TRACE("ncshm_take_hdl notificationHandle failed");
>>>>>> @@ -2695,7 +3042,7 @@ SaAisErrorT saNtfNotificationFilterFree(
>>>>>>         }
>>>>>>
>>>>>>         /* free the resources allocated by saNtf<ntfType>FilterAllocate 
>>>>>> */
>>>>>> -    ntfa_filter_hdl_rec_destructor(filter_hdl_rec);
>>>>>> +    ntfa_filter_destructor(filter_hdl_rec);
>>>>>>
>>>>>>         /** Delete the resources related to the notificationFilterHandle 
>>>>>> &
>>>>>>          *  remove reference in the client.
>>>>>> @@ -2729,7 +3076,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>>>>         TRACE_ENTER();
>>>>>>         SaAisErrorT rc = SA_AIS_ERR_NOT_EXIST;
>>>>>>         SaNtfHandleT ntfHandle;
>>>>>> -
>>>>>>         ntfa_client_hdl_rec_t *client_hdl_rec;
>>>>>>
>>>>>>         ntfsv_msg_t msg, *o_msg = NULL;
>>>>>> @@ -2737,7 +3083,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>>>>         ntfsv_unsubscribe_req_t *send_param;
>>>>>>         uint32_t timeout = NTFS_WAIT_TIME;
>>>>>>
>>>>>> -
>>>>>>         ntfHandle = ntfHandleGet(subscriptionId);
>>>>>>         if (ntfHandle == 0) {
>>>>>>             TRACE_1("ntfHandleGet failed, subscription not exist");
>>>>>> @@ -2753,9 +3098,16 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>>>>             goto done;
>>>>>>         }
>>>>>>
>>>>>> -    /**
>>>>>> -         ** Populate a sync MDS message
>>>>>> -         **/
>>>>>> +    /* Check NTF server availability */
>>>>>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>>>>> [Praveen] Here in both unavailability and down case, API will return
>>>>> without clean up of subscriber information. But README in patch 3 tals
>>>>> differently for the two cases.
>>>>> I think handling like saNtfFinalize() way needs to be done here.
>>>> [Minh]: I will have to check the history commit, to see if any reason
>>>> that agent doesn't silently release subscriber's resource here as
>>>> described in README. But it looks to me a bug, can I get back to you later?
>>> [Praveen] This needs to be corrected otherwise the same application will
>>> not be able to reuse old subscription_id. It will get ERR_EXIST.
>>>>>> +        ncshm_give_hdl(ntfHandle);
>>>>>> +        goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    if (client_hdl_rec->valid) {
>>>>>> +        /**
>>>>>> +          ** Populate a sync MDS message
>>>>>> +         **/
>>>>>>             memset(&msg, 0, sizeof(ntfsv_msg_t));
>>>>>>             msg.type = NTFSV_NTFA_API_MSG;
>>>>>>             msg.info.api_info.type = NTFSV_UNSUBSCRIBE_REQ;
>>>>>> @@ -2764,13 +3116,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>>>>             send_param->client_id = client_hdl_rec->ntfs_client_id;
>>>>>>             send_param->subscriptionId = subscriptionId;
>>>>>>
>>>>>> -        /* Check whether NTFS is up or not */
>>>>>> -        if (!ntfa_cb.ntfs_up) {
>>>>>> -            TRACE_1("NTFS down");
>>>>>> -            rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> -            goto done_give_hdl;
>>>>>> -        }
>>>>>> -
>>>>>>             /* Send a sync MDS message to obtain a log stream id */
>>>>>>             rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>>> timeout);
>>>>>>             if (rc != NCSCC_RC_SUCCESS) {
>>>>>> @@ -2785,6 +3130,7 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>>>>                 TRACE_1("Bad return status! rc = %d", rc);
>>>>>>                 goto done_give_hdl;
>>>>>>             }
>>>>>> +    }
>>>>>>             subscriberListItemRemove(subscriptionId);
>>>>>>
>>>>>>             /*Remove msg for subscriptionId from mailbox*/
>>>>>> @@ -2815,6 +3161,19 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>>>>      done_give_hdl:
>>>>>>         if (o_msg)
>>>>>>              ntfa_msg_destroy(o_msg);
>>>>>> +
>>>>>> +    if (!client_hdl_rec->valid) {
>>>>>> +        if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>>>>> +            if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>>>>> +                ncshm_give_hdl(ntfHandle);
>>>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>>> +                ntfa_shutdown(false);
>>>>>> +                goto done;
>>>>>> +            }
>>>>>> +        }
>>>>>> +    }
>>>>>>         ncshm_give_hdl(ntfHandle);
>>>>>>      done:
>>>>>>         TRACE_LEAVE();
>>>>>> @@ -2838,6 +3197,7 @@ SaAisErrorT saNtfNotificationReadInitial
>>>>>>         ntfsv_reader_init_req_2_t *send_param;
>>>>>>         uint32_t timeout = NTFS_WAIT_TIME;
>>>>>>
>>>>>> +    bool recovery_failed = false;
>>>>>>         TRACE_ENTER();
>>>>>>         if (notificationFilterHandles == NULL || readHandle == NULL) {
>>>>>>             rc = SA_AIS_ERR_INVALID_PARAM;
>>>>>> @@ -2872,6 +3232,18 @@ SaAisErrorT saNtfNotificationReadInitial
>>>>>>             goto done_give_client_hdl;
>>>>>>         }
>>>>>>
>>>>>> +    /* Check NTF server availability */
>>>>>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>>>>> +        goto done_give_client_hdl;
>>>>>> +
>>>>>> +    /* recovery */
>>>>>> +    if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
>>>>>> +        if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>>>>> +            recovery_failed = true;
>>>>>> +            goto done_give_client_hdl;
>>>>>> +        }
>>>>>> +    }
>>>>>> +
>>>>>>         /**
>>>>>>          ** Populate a sync MDS message
>>>>>>          **/
>>>>>> @@ -2884,13 +3256,6 @@ SaAisErrorT saNtfNotificationReadInitial
>>>>>>         send_param->head.searchCriteria = searchCriteria;
>>>>>>         send_param->f_rec = filters;
>>>>>>
>>>>>> -    /* Check whether NTFS is up or not */
>>>>>> -    if (!ntfa_cb.ntfs_up) {
>>>>>> -        TRACE("NTFS down");
>>>>>> -        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> -        goto done_give_client_hdl;
>>>>>> -    }
>>>>>> -
>>>>>>         /* Send a sync MDS message to obtain a log stream id */
>>>>>>         rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>>>>>         if (rc != NCSCC_RC_SUCCESS) {
>>>>>> @@ -2925,12 +3290,16 @@ SaAisErrorT saNtfNotificationReadInitial
>>>>>>         reader_hdl_rec->ntfHandle = ntfHandle;
>>>>>>         /* Store the readerId returned from server */
>>>>>>         reader_hdl_rec->reader_id =
>>>>>> o_msg->info.api_resp_info.param.reader_init_rsp.readerId;
>>>>>> +    memset(&reader_hdl_rec->filters, 0, sizeof(ntfsv_filter_ptrs_t));
>>>>>> +    ntfa_copy_ntf_filter_ptrs(&reader_hdl_rec->filters, &filters);
>>>>>> +    reader_hdl_rec->searchCriteria = searchCriteria;
>>>>>>         /**                  UnLock ntfa_CB            **/
>>>>>>         osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>>>
>>>>>>     done_give_client_hdl:
>>>>>>         if (o_msg)
>>>>>>             ntfa_msg_destroy(o_msg);
>>>>>> +
>>>>>>        if (client_hdl_rec)
>>>>>>             ncshm_give_hdl(client_hdl_rec->local_hdl);
>>>>>>         if (notificationFilterHandles) {
>>>>>> @@ -2947,6 +3316,12 @@ done_give_client_hdl:
>>>>>>         }
>>>>>>
>>>>>> ncshm_give_hdl(notificationFilterHandles->alarmFilterHandle);
>>>>>> +    if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
>>>>>> +        osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>>>> +        ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>>>>> +        osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>>> +        ntfa_shutdown(false);
>>>>>> +    }
>>>>>>     done:
>>>>>>         TRACE_LEAVE();
>>>>>>         return rc;
>>>>>> @@ -2983,42 +3358,46 @@ SaAisErrorT saNtfNotificationReadFinaliz
>>>>>>         }
>>>>>>         TRACE_1("reader_hdl_rec = %u", reader_hdl_rec->reader_hdl);
>>>>>>
>>>>>> -    /**
>>>>>> -     ** Populate a sync MDS message
>>>>>> -     **/
>>>>>> -    memset(&msg, 0, sizeof(ntfsv_msg_t));
>>>>>> -    msg.type = NTFSV_NTFA_API_MSG;
>>>>>> -    msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
>>>>>> -    send_param = &msg.info.api_info.param.reader_finalize;
>>>>>> -    send_param->client_id = client_hdl_rec->ntfs_client_id;
>>>>>> -    send_param->readerId = reader_hdl_rec->reader_id;
>>>>>> -
>>>>>> -    /* Check whether NTFS is up or not */
>>>>>> -    if (!ntfa_cb.ntfs_up) {
>>>>>> -        TRACE("NTFS down");
>>>>>> -        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> -        goto done_give_hdls;
>>>>>> +    /* Check NTF server availability */
>>>>>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>>>>> [Praveen] Same comment here as in Unsubscribe().
>>>>> [Minh] As above in Unsubscribed()
>>>>>> + ncshm_give_hdl(client_hdl_rec->local_hdl);
>>>>>> +        ncshm_give_hdl(readhandle);
>>>>>> +        goto done;
>>>>>>         }
>>>>>>
>>>>>> -    /* Send a sync MDS message */
>>>>>> -    rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>>>>> -    if (rc != NCSCC_RC_SUCCESS) {
>>>>>> -        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> -        goto done_give_hdls;
>>>>>> +    if (client_hdl_rec->valid) {
>>>>>> +        /**
>>>>>> +         ** Populate a sync MDS message
>>>>>> +         **/
>>>>>> +        memset(&msg, 0, sizeof(ntfsv_msg_t));
>>>>>> +        msg.type = NTFSV_NTFA_API_MSG;
>>>>>> +        msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
>>>>>> +        send_param = &msg.info.api_info.param.reader_finalize;
>>>>>> +        send_param->client_id = client_hdl_rec->ntfs_client_id;
>>>>>> +        send_param->readerId = reader_hdl_rec->reader_id;
>>>>>> +
>>>>>> +
>>>>>> +        /* Send a sync MDS message */
>>>>>> +        rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>>> timeout);
>>>>>> +        if (rc != NCSCC_RC_SUCCESS) {
>>>>>> +            rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> +            goto done_give_hdls;
>>>>>> +        }
>>>>>> +
>>>>>> +        osafassert(o_msg != NULL);
>>>>>> +        if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
>>>>>> +            rc = o_msg->info.api_resp_info.rc;
>>>>>> +            TRACE("Bad return status!!! rc = %d", rc);
>>>>>> +            goto done_give_hdls;
>>>>>> +        }
>>>>>> +
>>>>>> +        if (o_msg->info.api_resp_info.type !=
>>>>>> NTFSV_READER_FINALIZE_RSP) {
>>>>>> +            TRACE("msg type (%d) failed",
>>>>>> (int)o_msg->info.api_resp_info.type);
>>>>>> +            rc = SA_AIS_ERR_LIBRARY;
>>>>>> +            goto done_give_hdls;
>>>>>> +        }
>>>>>>         }
>>>>>>
>>>>>> -    osafassert(o_msg != NULL);
>>>>>> -    if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
>>>>>> -        rc = o_msg->info.api_resp_info.rc;
>>>>>> -        TRACE("Bad return status!!! rc = %d", rc);
>>>>>> -        goto done_give_hdls;
>>>>>> -    }
>>>>>> -
>>>>>> -    if (o_msg->info.api_resp_info.type !=
>>> NTFSV_READER_FINALIZE_RSP) {
>>>>>> -        TRACE("msg type (%d) failed",
>>>>>> (int)o_msg->info.api_resp_info.type);
>>>>>> -        rc = SA_AIS_ERR_LIBRARY;
>>>>>> -        goto done_give_hdls;
>>>>>> -    }
>>>>>>         osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>>>>         oas_rc = ntfa_reader_hdl_rec_del(&client_hdl_rec->reader_list,
>>>>>> reader_hdl_rec);
>>>>>>         if (oas_rc != NCSCC_RC_SUCCESS) {
>>>>>> @@ -3029,6 +3408,20 @@ SaAisErrorT saNtfNotificationReadFinaliz
>>>>>>      done_give_hdls:
>>>>>>         if (o_msg)
>>>>>>              ntfa_msg_destroy(o_msg);
>>>>>> +
>>>>>> +    if (!client_hdl_rec->valid) {
>>>>>> +        if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>>>>> +            if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>>>>> +                ncshm_give_hdl(client_hdl_rec->local_hdl);
>>>>>> +                ncshm_give_hdl(readhandle);
>>>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>>> +                ntfa_shutdown(false);
>>>>>> +                goto done;
>>>>>> +            }
>>>>>> +        }
>>>>>> +    }
>>>>>>         ncshm_give_hdl(client_hdl_rec->local_hdl);
>>>>>>      done_give_read_hdl:
>>>>>>         ncshm_give_hdl(readhandle);
>>>>>> @@ -3082,6 +3475,23 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>>>>>>         }
>>>>>>         TRACE_1("reader_hdl_rec = %u", reader_hdl_rec->reader_hdl);
>>>>>>
>>>>>> +    /* Check NTF server availability */
>>>>>> +    if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>>>>> +        goto done_give_hdls;
>>>>>> +
>>>>>> +    if (!client_hdl_rec->valid) {
>>>>>> +        if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>>>>> +            ncshm_give_hdl(client_hdl_rec->local_hdl);
>>>>>> +            ncshm_give_hdl(readHandle);
>>>>>> +            if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>>> +                ntfa_shutdown(false);
>>>>>> +            }
>>>>>> +            goto done;
>>>>>> +        }
>>>>>> +    }
>>>>>>         /**
>>>>>>          ** Populate a sync MDS message
>>>>>>          **/
>>>>>> @@ -3092,12 +3502,6 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>>>>>>         send_param->client_id = client_hdl_rec->ntfs_client_id;
>>>>>>         send_param->readerId = reader_hdl_rec->reader_id;
>>>>>>         send_param->searchDirection = searchDirection;
>>>>>> -    /* Check whether NTFS is up or not */
>>>>>> -    if (!ntfa_cb.ntfs_up) {
>>>>>> -        TRACE("NTFS down");
>>>>>> -        rc = SA_AIS_ERR_TRY_AGAIN;
>>>>>> -        goto done_give_hdls;
>>>>>> -    }
>>>>>>
>>>>>>         do {
>>>>>>             /* Send a sync MDS message */
>>>>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>>>>> b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>>>>> --- a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>>>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>>>>> @@ -369,7 +369,8 @@ static uint32_t ntfa_mds_svc_evt(struct
>>>>>>                 TRACE("NTFS down");
>>>>>>                 pthread_mutex_lock(&ntfa_cb.cb_lock);
>>>>>>                 memset(&ntfa_cb.ntfs_mds_dest, 0, sizeof(MDS_DEST));
>>>>>> -            ntfa_cb.ntfs_up = 0;
>>>>>> + ntfa_update_ntfsv_state(mds_cb_info->info.svc_evt.i_change ==
>>>>>> NCSMDS_NO_ACTIVE ?
>>>>>> +                    NTFA_NTFSV_NO_ACTIVE : NTFA_NTFSV_DOWN);
>>>>>>                 pthread_mutex_unlock(&ntfa_cb.cb_lock);
>>>>>>             }
>>>>>>             break;
>>>>>> @@ -382,7 +383,8 @@ static uint32_t ntfa_mds_svc_evt(struct
>>>>>>                 TRACE_2("MSG from NTFS NCSMDS_NEW_ACTIVE/UP");
>>>>>>                 pthread_mutex_lock(&ntfa_cb.cb_lock);
>>>>>>                 ntfa_cb.ntfs_mds_dest = mds_cb_info->info.svc_evt.i_dest;
>>>>>> -            ntfa_cb.ntfs_up = 1;
>>>>>> + ntfa_update_ntfsv_state(mds_cb_info->info.svc_evt.i_change ==
>>>>>> NCSMDS_NEW_ACTIVE ?
>>>>>> +                    NTFA_NTFSV_NEW_ACTIVE : NTFA_NTFSV_UP);
>>>>>>                 if (ntfa_cb.ntfs_sync_awaited) {
>>>>>>                     /* signal waiting thread */
>>>>>>                     m_NCS_SEL_OBJ_IND(&ntfa_cb.ntfs_sync_sel);
>>>>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>>>>> b/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>>>>> --- a/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>>>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>>>>> @@ -49,7 +49,7 @@ static unsigned int ntfa_create(void)
>>>>>>         }
>>>>>>
>>>>>>         /* Block and wait for indication from MDS meaning NTFS is up */
>>>>>> -
>>> osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_sync_sel),
>>> 30000);
>>>>>> +
>>> osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_sync_sel),
>>> 10000);
>>>>> [Praveen] Did not get why it is changed to 10 seconds.
>>>>> [Minh] I think 30 seconds is just too long
>>>>>>         pthread_mutex_lock(&ntfa_cb.cb_lock);
>>>>>>         ntfa_cb.ntfs_sync_awaited = 0;
>>>>>> @@ -108,7 +108,25 @@ static bool ntfa_clear_mbx(NCSCONTEXT ar
>>>>>>         }
>>>>>>         return true;
>>>>>>     }
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : ntfa_notification_list_del
>>>>>>
>>>>>> +  Description   : This routine free element allocated in list of
>>>>>> notification
>>>>>> +
>>>>>> +  Arguments     : pointer to the list of notification records anchor.
>>>>>> +
>>>>>> +  Return Values : None
>>>>>> +
>>>>>> +  Notes         :
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +static void ntfa_notification_list_del(ntfa_notification_hdl_rec_t
>>>>>> **plstr_hdl)
>>>>>> +{
>>>>>> +    ntfa_notification_hdl_rec_t *lstr_hdl = *plstr_hdl;
>>>>>> +    while (lstr_hdl != NULL) {
>>>>>> +        ntfa_notification_destructor(lstr_hdl);
>>>>>> +        lstr_hdl = lstr_hdl->next;
>>>>>> +    }
>>>>>> +}
>>>>>>
>>> /**********************************************************
>>> ******************
>>>>>>       Name          : ntfa_notification_hdl_rec_list_del
>>>>>>
>>>>>> @@ -130,6 +148,70 @@ static void ntfa_notification_hdl_rec_li
>>>>>>             lstr_hdl = NULL;
>>>>>>         }
>>>>>>     }
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : ntfa_filter_list_del
>>>>>> +
>>>>>> +  Description   : This routine free element allocated in list of filter
>>>>>> +
>>>>>> +  Arguments     : pointer to the list of filters records anchor.
>>>>>> +
>>>>>> +  Return Values : None
>>>>>> +
>>>>>> +  Notes         :
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +static void ntfa_filter_list_del(ntfa_filter_hdl_rec_t **plstr_hdl)
>>>>>> +{
>>>>>> +    ntfa_filter_hdl_rec_t *lstr_hdl = *plstr_hdl;
>>>>>> +
>>>>>> +    while (lstr_hdl != NULL) {
>>>>>> +        ntfa_filter_destructor(lstr_hdl);
>>>>>> +        lstr_hdl = lstr_hdl->next;
>>>>>> +    }
>>>>>> +}
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : ntfa_filter_hdl_rec_list_del
>>>>>> +
>>>>>> +  Description   : This routine deletes a list of allocated filters
>>>>>> +
>>>>>> +  Arguments     : pointer to the list of filters records anchor.
>>>>>> +
>>>>>> +  Return Values : None
>>>>>> +
>>>>>> +  Notes         :
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +static void ntfa_filter_hdl_rec_list_del(ntfa_filter_hdl_rec_t
>>>>>> **plstr_hdl)
>>>>>> +{
>>>>>> +    ntfa_filter_hdl_rec_t *lstr_hdl;
>>>>>> +
>>>>>> +    while ((lstr_hdl = *plstr_hdl) != NULL) {
>>>>>> +        *plstr_hdl = lstr_hdl->next;
>>>>>> +        ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, lstr_hdl->filter_hdl);
>>>>>> +        free(lstr_hdl);
>>>>>> +        lstr_hdl = NULL;
>>>>>> +    }
>>>>>> +}
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : ntfa_reader_hdl_rec_list_del
>>>>>> +
>>>>>> +  Description   : This routine deletes a list of allocated readers
>>>>>> +
>>>>>> +  Arguments     : pointer to the list of readers records anchor.
>>>>>> +
>>>>>> +  Return Values : None
>>>>>> +
>>>>>> +  Notes         :
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +static void ntfa_reader_hdl_rec_list_del(ntfa_reader_hdl_rec_t
>>>>>> **plstr_hdl)
>>>>>> +{
>>>>>> +    ntfa_reader_hdl_rec_t *lstr_hdl;
>>>>>> +    while ((lstr_hdl = *plstr_hdl) != NULL) {
>>>>>> +        *plstr_hdl = lstr_hdl->next;
>>>>>> +        ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, lstr_hdl-
>>>> reader_hdl);
>>>>>> +        ntfa_del_ntf_filter_ptrs(&lstr_hdl->filters);
>>>>>> +        free(lstr_hdl);
>>>>>> +        lstr_hdl = NULL;
>>>>>> +    }
>>>>>> +}
>>>>>>
>>>>>>     static SaAisErrorT
>>>>>> ntfa_alloc_callback_notification(SaNtfNotificationsT *notification,
>>>>>>                                 ntfsv_send_not_req_t *not_cbk,
>>>>>> ntfa_client_hdl_rec_t *hdl_rec)
>>>>>> @@ -407,6 +489,10 @@ static SaAisErrorT ntfa_hdl_cbk_rec_prc(
>>>>>> free(cbk_info->param.discarded_cbk.discardedNotificationIdentifiers);
>>>>>>             }
>>>>>>             break;
>>>>>> +    case NTFSV_DUMMY_CALLBACK:
>>>>>> +        TRACE("Do nothing with dummy callback, just return OK");
>>>>>> +        rc = SA_AIS_OK;
>>>>>> +        break;
>>>>>>         default:
>>>>>>             TRACE("unsupported callback type: %d", cbk_info->type);
>>>>>>             rc = SA_AIS_ERR_LIBRARY;
>>>>>> @@ -544,20 +630,21 @@ unsigned int ntfa_startup(void)
>>>>>>      *
>>>>>>      * @return unsigned int
>>>>>>      */
>>>>>> -unsigned int ntfa_shutdown(void)
>>>>>> +unsigned int ntfa_shutdown(bool forced)
>>>>>>     {
>>>>>>         unsigned int rc = NCSCC_RC_SUCCESS;
>>>>>>
>>>>>> -    TRACE_ENTER2("ntfa_use_count: %u", ntfa_use_count);
>>>>>> +    TRACE_ENTER2("ntfa_use_count: %u, forced: %u", ntfa_use_count,
>>>>>> forced);
>>>>>>         pthread_mutex_lock(&ntfa_lock);
>>>>>>
>>>>>> -    if (ntfa_use_count > 1) {
>>>>>> -        /* Users still exist, just decrement the use count */
>>>>>> -        ntfa_use_count--;
>>>>>> -    } else if (ntfa_use_count == 1) {
>>>>>> +    if ((forced && (ntfa_use_count > 0)) || (ntfa_use_count == 1)) {
>>>>>>             ntfa_destroy();
>>>>>>             rc = ncs_agents_shutdown();
>>>>>>             ntfa_use_count = 0;
>>>>>> +        ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_NONE;
>>>>>> +    } else if (ntfa_use_count > 1) {
>>>>>> +        /* Users still exist, just decrement the use count */
>>>>>> +        ntfa_use_count--;
>>>>>>         }
>>>>>>
>>>>>>         pthread_mutex_unlock(&ntfa_lock);
>>>>>> @@ -633,7 +720,27 @@ ntfa_client_hdl_rec_t *ntfa_find_hdl_rec
>>>>>>
>>>>>>         return NULL;
>>>>>>     }
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : ntfa_subscriber_list_del
>>>>>>
>>>>>> +  Description   : This routine deletes a list of allocated subscribers
>>>>>> +
>>>>>> +  Arguments     : pointer to the list of subscribers records anchor.
>>>>>> +
>>>>>> +  Return Values : None
>>>>>> +
>>>>>> +  Notes         :
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +void ntfa_subscriber_list_del()
>>>>>> +{
>>>>>> +    ntfa_subscriber_list_t *listPtr = subscriberNoList;
>>>>>> +    while (listPtr != NULL) {
>>>>>> +        ntfa_subscriber_list_t* tmpSub = listPtr;
>>>>>> +        listPtr = listPtr->next;
>>>>>> +        free(tmpSub);
>>>>>> +    }
>>>>>> +    subscriberNoList = NULL;
>>>>>> +}
>>>>>>
>>> /**********************************************************
>>> ******************
>>>>>>       Name          : ntfa_hdl_list_del
>>>>>>
>>>>>> @@ -651,15 +758,24 @@ void ntfa_hdl_list_del(ntfa_client_hdl_r
>>>>>>
>>>>>>         while ((client_hdl = *p_client_hdl) != NULL) {
>>>>>>             *p_client_hdl = client_hdl->next;
>>>>>> +        m_NCS_IPC_DETACH(&client_hdl->mbx, ntfa_clear_mbx, NULL);
>>>>>> +        m_NCS_IPC_RELEASE(&client_hdl->mbx, NULL);
>>>>>>             ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, client_hdl-
>>>> local_hdl);
>>>>>>         /** clean up the channel records for this ntfa-client
>>>>>>              **/
>>>>>> + ntfa_notification_list_del(&client_hdl->notification_list);
>>>>>> ntfa_notification_hdl_rec_list_del(&client_hdl->notification_list);
>>>>>> +
>>>>>> +        ntfa_filter_list_del(&client_hdl->filter_list);
>>>>>> + ntfa_filter_hdl_rec_list_del(&client_hdl->filter_list);
>>>>>> +
>>>>>> + ntfa_reader_hdl_rec_list_del(&client_hdl->reader_list);
>>>>>>         /** remove the association with hdl-mngr
>>>>>>              **/
>>>>>>             free(client_hdl);
>>>>>>             client_hdl = 0;
>>>>>>         }
>>>>>> +    ntfa_subscriber_list_del();
>>>>>>     }
>>>>>>
>>>>>>
>>> /**********************************************************
>>> ******************
>>>>>> @@ -757,10 +873,80 @@ uint32_t ntfa_filter_hdl_rec_del(ntfa_fi
>>>>>>         TRACE("The node couldn't be deleted");
>>>>>>         return NCSCC_RC_FAILURE;
>>>>>>     }
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : ntfa_hdl_rec_force_del
>>>>>>
>>>>>> +  Description   : This routine deletes all memory allocated to client.
>>>>>> +
>>>>>> +  Arguments     : NTFA_CLIENT_HDL_REC **list_head
>>>>>> +          NTFA_CLIENT_HDL_REC *rm_node
>>>>>> +
>>>>>> +  Return Values : None
>>>>>> +
>>>>>> +  Notes         :
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **list_head,
>>>>>> ntfa_client_hdl_rec_t *rm_node)
>>>>>> +{
>>>>>> +    ntfa_client_hdl_rec_t *list_iter = *list_head;
>>>>>> +    TRACE_ENTER();
>>>>>> +    /* First remove the rm_node out of the list of client */
>>>>>> +    if (list_iter == rm_node)
>>>>>> +        *list_head = rm_node->next;
>>>>>> +    else {
>>>>>> +        while (list_iter) {
>>>>>> +            if (list_iter->next == rm_node) {
>>>>>> +                list_iter->next = rm_node->next;
>>>>>> +                break;
>>>>>> +            }
>>>>>> +            list_iter = list_iter->next;
>>>>>> +        }
>>>>>> +    }
>>>>>> +    /* Release all msgs in mailbox */
>>>>>> +    ntfsv_msg_t *cbk_msg;
>>>>>> +    while((cbk_msg =
>>>>>> (ntfsv_msg_t*)m_NCS_IPC_NON_BLK_RECEIVE(&rm_node->mbx,
>>> cbk_msg))
>>>>>> +        != NULL) {
>>>>>> +        ntfa_msg_destroy(cbk_msg);
>>>>>> +    }
>>>>>> +    /* delete subscriber of this client out of the subcriberNoList*/
>>>>>> +    ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
>>>>>> +    while (subscriber_hdl != NULL) {
>>>>>> +        ntfa_subscriber_list_t *rm_subscriber = subscriber_hdl;
>>>>>> +        subscriber_hdl = subscriber_hdl->next;
>>>>>> +        if (rm_node->local_hdl ==
>>>>>> rm_subscriber->subscriberListNtfHandle) {
>>>>>> +            if (rm_subscriber->next != NULL) {
>>>>>> +                rm_subscriber->next->prev = rm_subscriber->prev;
>>>>>> +            }
>>>>>> +
>>>>>> +            if (rm_subscriber->prev != NULL) {
>>>>>> +                rm_subscriber->prev->next = rm_subscriber->next;
>>>>>> +            } else {
>>>>>> +                if (rm_subscriber->next != NULL)
>>>>>> +                    subscriberNoList = rm_subscriber->next;
>>>>>> +                else
>>>>>> +                    subscriberNoList = NULL;
>>>>>> +            }
>>>>>> + ntfa_del_ntf_filter_ptrs(&rm_subscriber->filters);
>>>>>> +            free(rm_subscriber);
>>>>>> +        }
>>>>>> +    }
>>>>>> +    /* Now delete client */
>>>>>> +    m_NCS_IPC_DETACH(&rm_node->mbx, ntfa_clear_mbx, NULL);
>>>>>> +    m_NCS_IPC_RELEASE(&rm_node->mbx, NULL);
>>>>>> +    ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node->local_hdl);
>>>>>> + ntfa_notification_list_del(&rm_node->notification_list);
>>>>>> + ntfa_notification_hdl_rec_list_del(&rm_node->notification_list);
>>>>>> +
>>>>>> +    ntfa_filter_list_del(&rm_node->filter_list);
>>>>>> +    ntfa_filter_hdl_rec_list_del(&rm_node->filter_list);
>>>>>> +
>>>>>> +    ntfa_reader_hdl_rec_list_del(&rm_node->reader_list);
>>>>>> +    free(rm_node);
>>>>>> +
>>>>>> +    TRACE_LEAVE();
>>>>>> +}
>>>>>>
>>> /**********************************************************
>>> ******************
>>>>>>       Name          : ntfa_hdl_rec_del
>>>>>> -
>>>>>> +
>>>>>>       Description   : This routine deletes the a client handle record 
>>>>>> from
>>>>>>                       a list of client hdl records.
>>>>>>
>>>>>> @@ -962,7 +1148,7 @@ ntfa_client_hdl_rec_t *ntfa_hdl_rec_add(
>>>>>>         /** Associate with the client_id obtained from NTFS
>>>>>>          **/
>>>>>>         rec->ntfs_client_id = client_id;
>>>>>> -
>>>>>> +    rec->valid = true;
>>>>>>         /** Initialize and attach the IPC/Priority queue
>>>>>>          **/
>>>>>>
>>>>>> @@ -1063,7 +1249,7 @@ static void logtrace_init_constructor(vo
>>>>>>      *
>>>>>>      * @param instance
>>>>>>      */
>>>>>> -void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t *instance)
>>>>>> +void ntfa_notification_destructor(ntfa_notification_hdl_rec_t
>>>>>> *instance)
>>>>>>     {
>>>>>>         ntfa_notification_hdl_rec_t *notificationInstance = instance;
>>>>>>
>>>>>> @@ -1111,7 +1297,7 @@ void ntfa_hdl_rec_destructor(ntfa_notifi
>>>>>>      *
>>>>>>      * @param instance
>>>>>>      */
>>>>>> -void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t *filter_rec)
>>>>>> +void ntfa_filter_destructor(ntfa_filter_hdl_rec_t *filter_rec)
>>>>>>     {
>>>>>>         switch (filter_rec->ntfType) {
>>>>>>         case SA_NTF_TYPE_OBJECT_CREATE_DELETE:
>>>>>> @@ -1202,6 +1388,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>>>>>>              **/
>>>>>>             ncshm_give_hdl(rm_node->reader_hdl);
>>>>>>             ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node-
>>>> reader_hdl);
>>>>>> +        ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>>>>>>             free(rm_node);
>>>>>>             return NCSCC_RC_SUCCESS;
>>>>>>         } else {        /* find the rec */
>>>>>> @@ -1213,6 +1400,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>>>>>>                      **/
>>>>>>                     ncshm_give_hdl(rm_node->reader_hdl);
>>>>>>                     ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA,
>>>>>> rm_node->reader_hdl);
>>>>>> + ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>>>>>>                     free(rm_node);
>>>>>>                     return NCSCC_RC_SUCCESS;
>>>>>>                 }
>>>>>> @@ -1258,3 +1446,258 @@ void ntfa_add_to_async_cbk_msg_list(ntfs
>>>>>>
>>>>>>         TRACE_LEAVE();
>>>>>>     }
>>>>>> +
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : ntfa_notify_handle_invalid
>>>>>> +
>>>>>> +  Description   : This routine sends a dummy callback msg to
>>>>>> client's mailbox
>>>>>> +          so that the client polls in and calls saNtfDispatch.
>>>>>> +  Arguments     :
>>>>>> +
>>>>>> +  Return Values : None
>>>>>> +
>>>>>> +  Notes         : None
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +void ntfa_notify_handle_invalid() {
>>>>>> +    ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
>>>>>> +    TRACE_ENTER();
>>>>>> +    while (client_hdl != NULL) {
>>>>>> +        /* Only applicable for subscriber */
>>>>>> +        if (client_hdl->reg_cbk.saNtfNotificationCallback != NULL ||
>>>>>> + client_hdl->reg_cbk.saNtfNotificationDiscardedCallback != NULL) {
>>>>>> +            /* Create a dummy msg */
>>>>>> +            ntfsv_msg_t *msg = malloc(sizeof(ntfsv_msg_t));
>>>>>> +            memset(msg, 0, sizeof(ntfsv_msg_t));
>>>>>> +            msg->info.cbk_info.type = NTFSV_DUMMY_CALLBACK;
>>>>>> +            /* Send dummy msg to client mailbox */
>>>>> [Praveen] Is this dummy callback being sent to client in a way to
>>>>> force it to recover client and subscriber?
>>>> [Minh] Because agent doesn't recover client in mds callback, so in case
>>>> of subscriber who is just waiting for notification, agent needs to send
>>>> dummy callback to client's mailbox to wake up client's polling loop.
>>>> Then client will call saAmfDispatch, and from there agent will recover
>>>> client handles. This dummy callback is invisible to client.
>>>>>> +            if (m_NCS_IPC_SEND(&client_hdl->mbx, msg,
>>>>>> MDS_SEND_PRIORITY_HIGH) != NCSCC_RC_SUCCESS) {
>>>>>> +                TRACE_1("m_NCS_IPC_SEND Failed to client(id:%u)",
>>>>>> client_hdl->ntfs_client_id);
>>>>>> +                ntfa_msg_destroy(msg);
>>>>>> +            }
>>>>>> +        }
>>>>>> +
>>>>>> +        client_hdl = client_hdl->next;
>>>>>> +    }
>>>>>> +    TRACE_LEAVE();
>>>>>> +}
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : ntfa_update_ntfsv_state
>>>>>> +
>>>>>> +  Description   : Update current NTF Server state by the
>>>>>> @changedState indicated
>>>>>> +          by MDS event
>>>>>> +  Arguments     : @changedState [IN]: state to be changed of NTF Server
>>>>>> +
>>>>>> +  Return Values : None
>>>>>> +
>>>>>> +  Notes         : None
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState)
>>>>>> +{
>>>>>> +    TRACE_ENTER();
>>>>>> +    TRACE_1("Current state: %u, Changed state: %u",
>>>>>> ntfa_cb.ntfa_ntfsv_state,
>>>>>> +                                                changedState);
>>>>>> +
>>>>>> +    ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
>>>>>> +
>>>>>> +    switch (ntfa_cb.ntfa_ntfsv_state){
>>>>>> +    case NTFA_NTFSV_NONE:
>>>>>> +        ntfa_cb.ntfa_ntfsv_state = changedState;
>>>>>> +        break;
>>>>>> +    case NTFA_NTFSV_DOWN:
>>>>>> +        if (changedState == NTFA_NTFSV_NEW_ACTIVE ||
>>>>>> +            changedState == NTFA_NTFSV_UP) {
>>>>>> +            TRACE("Active NTF server has been restarted");
>>>>>> +            ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_UP;
>>>>>> +            ntfa_notify_handle_invalid();
>>>>>> +        } else
>>>>>> +            TRACE("Unexpected state changes");
>>>>>> +        break;
>>>>>> +    case NTFA_NTFSV_NO_ACTIVE:
>>>>>> +        if (changedState == NTFA_NTFSV_NEW_ACTIVE) {
>>>>>> +            TRACE("Standby NTF server becomes new Active");
>>>>>> +            /* NTF server is functioning normally */
>>>>>> +            ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_UP;
>>>>>> +        } else if (changedState == NTFA_NTFSV_DOWN) {
>>>>>> +            TRACE("Active NTF Server is Down");
>>>>>> +            ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_DOWN;
>>>>>> +            /* Mark all client handles are invalid */
>>>>>> +            while (client_hdl != NULL) {
>>>>>> +                client_hdl->valid = false;
>>>>>> +                client_hdl = client_hdl->next;
>>>>>> +            }
>>>>>> +        }
>>>>>> +        break;
>>>>>> +    case NTFA_NTFSV_NEW_ACTIVE:
>>>>>> +        TRACE("Unknown");
>>>>>> +        break;
>>>>>> +    case NTFA_NTFSV_UP:
>>>>>> +        if (changedState == NTFA_NTFSV_NO_ACTIVE) {
>>>>>> +            TRACE("Active NTF server temporarily unavailable");
>>>>>> +            /* Failover/Switchover is happening
>>>>>> +             * Any API calls result in TRY_AGAIN
>>>>>> +             */
>>>>>> +            ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_NO_ACTIVE;
>>>>>> +        } else
>>>>>> +            TRACE("Unexpected state changes");
>>>>>> +        break;
>>>>>> +    default:
>>>>>> +        osafassert(false);
>>>>>> +    }
>>>>>> +
>>>>>> +    TRACE_LEAVE();
>>>>>> +}
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : ntfa_copy_ntf_filter_ptrs
>>>>>> +
>>>>>> +  Description   : Copy a list of filter from pSrc to pDes
>>>>>> +
>>>>>> +  Arguments     : pDes* [OUT]: list of outcome filter
>>>>>> +          pSrc* [IN]: list of input filter
>>>>>> +
>>>>>> +  Return Values : SA_AIS_OK if succeed, other values as failed
>>>>>> +
>>>>>> +  Notes         : None
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
>>>>>> +                                const ntfsv_filter_ptrs_t* pSrc) {
>>>>>> +    SaAisErrorT rc = SA_AIS_OK;
>>>>>> +    SaNtfNotificationFilterHeaderT *des_header;
>>>>>> +    SaNtfNotificationFilterHeaderT *src_header;
>>>>>> +    TRACE_ENTER();
>>>>>> +
>>>>>> +    if (pSrc->alarm_filter) {
>>>>>> +        pDes->alarm_filter = calloc(1,
>>>>>> sizeof(SaNtfAlarmNotificationFilterT));
>>>>>> +        des_header = &(pDes->alarm_filter->notificationFilterHeader);
>>>>>> +        src_header = &(pSrc->alarm_filter->notificationFilterHeader);
>>>>>> +        if ((rc = ntfsv_filter_header_alloc(des_header,
>>>>>> src_header->numEventTypes,
>>>>>> + src_header->numNotificationObjects,
>>>>>> + src_header->numNotifyingObjects,
>>>>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +        if ((rc = ntfsv_filter_alarm_alloc(pDes->alarm_filter,
>>>>>> + pSrc->alarm_filter->numProbableCauses,
>>>>>> + pSrc->alarm_filter->numPerceivedSeverities,
>>>>>> + pSrc->alarm_filter->numTrends)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +        if ((rc = ntfsv_copy_ntf_filter_alarm(pDes->alarm_filter,
>>>>>> + pSrc->alarm_filter)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    if (pSrc->sec_al_filter) {
>>>>>> +        pDes->sec_al_filter = calloc(1,
>>>>>> sizeof(SaNtfSecurityAlarmNotificationFilterT));
>>>>>> +        des_header = &(pDes->sec_al_filter->notificationFilterHeader);
>>>>>> +        src_header = &(pSrc->sec_al_filter->notificationFilterHeader);
>>>>>> +        if ((rc = ntfsv_filter_header_alloc(des_header,
>>>>>> src_header->numEventTypes,
>>>>>> + src_header->numNotificationObjects,
>>>>>> + src_header->numNotifyingObjects,
>>>>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +        if ((rc = ntfsv_filter_sec_alarm_alloc(pDes->sec_al_filter,
>>>>>> + pSrc->sec_al_filter->numProbableCauses,
>>>>>> + pSrc->sec_al_filter->numSeverities,
>>>>>> + pSrc->sec_al_filter->numSecurityAlarmDetectors,
>>>>>> + pSrc->sec_al_filter->numServiceUsers,
>>>>>> + pSrc->sec_al_filter->numServiceProviders)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +        if ((rc = ntfsv_copy_ntf_filter_sec_alarm(pDes->sec_al_filter,
>>>>>> + pSrc->sec_al_filter)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    if (pSrc->sta_ch_filter) {
>>>>>> +        pDes->sta_ch_filter = calloc(1,
>>>>>> sizeof(SaNtfStateChangeNotificationFilterT));
>>>>>> +        des_header = &(pDes->sta_ch_filter->notificationFilterHeader);
>>>>>> +        src_header = &(pSrc->sta_ch_filter->notificationFilterHeader);
>>>>>> +        if ((rc = ntfsv_filter_header_alloc(des_header,
>>>>>> src_header->numEventTypes,
>>>>>> + src_header->numNotificationObjects,
>>>>>> + src_header->numNotifyingObjects,
>>>>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +        if ((rc = ntfsv_filter_state_ch_alloc(pDes->sta_ch_filter,
>>>>>> + pSrc->sta_ch_filter->numSourceIndicators,
>>>>>> + pSrc->sta_ch_filter->numStateChanges)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +        if ((rc = ntfsv_copy_ntf_filter_state_ch(pDes->sta_ch_filter,
>>>>>> + pSrc->sta_ch_filter)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    if (pSrc->obj_cr_del_filter) {
>>>>>> +        pDes->obj_cr_del_filter = calloc(1,
>>>>>> sizeof(SaNtfObjectCreateDeleteNotificationFilterT));
>>>>>> +        des_header =
>>>>>> &(pDes->obj_cr_del_filter->notificationFilterHeader);
>>>>>> +        src_header =
>>>>>> &(pSrc->obj_cr_del_filter->notificationFilterHeader);
>>>>>> +        if ((rc = ntfsv_filter_header_alloc(des_header,
>>>>>> src_header->numEventTypes,
>>>>>> + src_header->numNotificationObjects,
>>>>>> + src_header->numNotifyingObjects,
>>>>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +        if ((rc =
>>>>>> ntfsv_filter_obj_cr_del_alloc(pDes->obj_cr_del_filter,
>>>>>> + pSrc->obj_cr_del_filter->numSourceIndicators)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +        if ((rc =
>>>>>> ntfsv_copy_ntf_filter_obj_cr_del(pDes->obj_cr_del_filter,
>>>>>> + pSrc->obj_cr_del_filter)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +    }
>>>>>> +
>>>>>> +    if (pSrc->att_ch_filter) {
>>>>>> +        pDes->att_ch_filter = calloc(1,
>>>>>> sizeof(SaNtfAttributeChangeNotificationFilterT));
>>>>>> +        des_header = &(pDes->att_ch_filter->notificationFilterHeader);
>>>>>> +        src_header = &(pSrc->att_ch_filter->notificationFilterHeader);
>>>>>> +        if ((rc = ntfsv_filter_header_alloc(des_header,
>>>>>> src_header->numEventTypes,
>>>>>> + src_header->numNotificationObjects,
>>>>>> + src_header->numNotifyingObjects,
>>>>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +        if ((rc = ntfsv_filter_attr_change_alloc(pDes->att_ch_filter,
>>>>>> + pSrc->att_ch_filter->numSourceIndicators)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +        if ((rc = ntfsv_copy_ntf_filter_attr_ch(pDes->att_ch_filter,
>>>>>> + pSrc->att_ch_filter)) != SA_AIS_OK)
>>>>>> +            goto done;
>>>>>> +    }
>>>>>> +done:
>>>>>> +    TRACE_LEAVE();
>>>>>> +    return rc;
>>>>>> +}
>>>>>> +
>>>>>>
>>> +/*********************************************************
>>> *******************
>>>>>> +  Name          : ntfa_del_ntf_filter_ptrs
>>>>>> +
>>>>>> +  Description   : Delete the filter pointers
>>>>>> +
>>>>>> +  Arguments     : filter_ptrs* [IN/OUT]: filter pointers
>>>>>> +
>>>>>> +  Return Values : SA_AIS_OK if succeed, other values as failed
>>>>>> +
>>>>>> +  Notes         : None
>>>>>>
>>> +*********************************************************
>>> *********************/
>>>>>> +SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t* filter_ptrs)
>>>>>> +{
>>>>>> +    SaAisErrorT rc = SA_AIS_OK;
>>>>>> +    if (filter_ptrs->alarm_filter) {
>>>>>> +        ntfsv_filter_alarm_free(filter_ptrs->alarm_filter, true);
>>>>>> +        free(filter_ptrs->alarm_filter);
>>>>>> +    }
>>>>>> +
>>>>>> +    if (filter_ptrs->sec_al_filter) {
>>>>>> + ntfsv_filter_sec_alarm_free(filter_ptrs->sec_al_filter, true);
>>>>>> +        free(filter_ptrs->sec_al_filter);
>>>>>> +    }
>>>>>> +
>>>>>> +    if (filter_ptrs->sta_ch_filter) {
>>>>>> + ntfsv_filter_state_ch_free(filter_ptrs->sta_ch_filter, true);
>>>>>> +        free(filter_ptrs->sta_ch_filter);
>>>>>> +    }
>>>>>> +
>>>>>> +    if (filter_ptrs->obj_cr_del_filter) {
>>>>>> + ntfsv_filter_obj_cr_del_free(filter_ptrs->obj_cr_del_filter, true);
>>>>>> +        free(filter_ptrs->obj_cr_del_filter);
>>>>>> +    }
>>>>>> +
>>>>>> +    if (filter_ptrs->att_ch_filter) {
>>>>>> + ntfsv_filter_attr_ch_free(filter_ptrs->att_ch_filter, true);
>>>>>> +        free(filter_ptrs->att_ch_filter);
>>>>>> +    }
>>>>>> +    return rc;
>>>>>> +}
>>>>>>
> ------------------------------------------------------------------------------
> Site24x7 APM Insight: Get Deep Visibility into Application Performance
> APM + Mobile APM + RUM: Monitor 3 App instances at just $35/Month
> Monitor end-to-end web transactions and take corrective actions now
> Troubleshoot faster and improve end-user experience. Signup Now!
> http://pubads.g.doubleclick.net/gampad/clk?id=272487151&iu=/4140
> _______________________________________________
> Opensaf-devel mailing list
> Opensaf-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/opensaf-devel
>


------------------------------------------------------------------------------
Site24x7 APM Insight: Get Deep Visibility into Application Performance
APM + Mobile APM + RUM: Monitor 3 App instances at just $35/Month
Monitor end-to-end web transactions and take corrective actions now
Troubleshoot faster and improve end-user experience. Signup Now!
http://pubads.g.doubleclick.net/gampad/clk?id=272487151&iu=/4140
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel

Reply via email to