Please see the responses with [Praveen]
Thanks,
Praveen
On 12-Feb-16 8:52 PM, minh chau wrote:
> Hi Praveen,
>
> Please find my comments inline [Minh]
>
> Thanks,
> Minh
>
> On 12/02/16 22:10, praveen malviya wrote:
>> Hi Minh,
>>
>> Please find some initial comments and questions marked with [Praveen].
>>
>> One question regarding the approach.
>> In most of the changed APIs from saNtfInitialize() to
>> saNtfNotificationReadFinalize(), clients are being recovered. Why the
>> idea of recovering them is not considered in mds callback itself in
>> the function call ntfa_update_ntfsv_state() when server state changes
>> from DOWN to UP. It will save changes in APIs for recovering the
>> clients. Clients are being marked invalid in the mds_callback itself.
>>
> [Minh] From memory I did try to recover clients in mds callback
> (svc_evt) where indicates server UP. Though recovery needs to send
> messages (reintroducing client id, read id, ...) to server, that will
> end up another mds enc callback inside of first callback. That results
> into fail to send *recovery* messages, there are two mds callback but
> only one return. To solve this, I need to start a thread the recover
> clients in background. That would cause another problem where up calls
> from client could be blocked, since the recovery thread needs to block
> mutual resource handles.
> Another realistic reason, client can give up to send/read/receive
> notification after receiving couple times of TRY_AGAIN due to server
> down, and doing nothing until finalize handle. And after client stops
> TRY_AGAIN loop, server can be restarted. In such cases, no needs to
> recover client handles.
>> Thanks,
>> Praveen
>>
>> On 23-Dec-15 9:32 AM, Minh Hon Chau wrote:
>>> osaf/libs/agents/saf/ntfa/ntfa.h | 31 +-
>>> osaf/libs/agents/saf/ntfa/ntfa_api.c | 672
>>> +++++++++++++++++++++++++++------
>>> osaf/libs/agents/saf/ntfa/ntfa_mds.c | 6 +-
>>> osaf/libs/agents/saf/ntfa/ntfa_util.c | 465 ++++++++++++++++++++++-
>>> 4 files changed, 1022 insertions(+), 152 deletions(-)
>>>
>>>
>>> The patch contains support for cloud resilience feature
>>> in NTF Agent code. Please refer README.HYDRA for content
>>> of the changes
>>>
>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa.h
>>> b/osaf/libs/agents/saf/ntfa/ntfa.h
>>> --- a/osaf/libs/agents/saf/ntfa/ntfa.h
>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa.h
>>> @@ -91,6 +91,7 @@ typedef struct ntfa_filter_hdl_rec {
>>> typedef struct subscriberList {
>>> SaNtfHandleT subscriberListNtfHandle;
>>> SaNtfSubscriptionIdT subscriberListSubscriptionId;
>>> + ntfsv_filter_ptrs_t filters; /* remember the filters used by
>>> this subscriber */
>>> struct subscriberList *prev;
>>> struct subscriberList *next;
>>> } ntfa_subscriber_list_t;
>>> @@ -100,6 +101,10 @@ typedef struct ntfa_reader_hdl_rec {
>>> unsigned int reader_id; /* handle value returned by NTFS for
>>> this client */
>>> SaNtfHandleT ntfHandle;
>>> unsigned int reader_hdl; /* READER handle from handle mgr */
>>> +
>>> + ntfsv_filter_ptrs_t filters; /* remember the filters used by
>>> this reader */
>>> + SaNtfSearchCriteriaT searchCriteria; /* remember the
>>> searchCriteria for recovery */
>>> +
>>> struct ntfa_reader_hdl_rec *next; /* next pointer for the
>>> list in ntfa_cb_t */
>>> struct ntfa_client_hdl_rec *parent_hdl; /* Back Pointer to
>>> the client instantiation */
>>> } ntfa_reader_hdl_rec_t;
>>> @@ -114,24 +119,35 @@ typedef struct ntfa_client_hdl_rec {
>>> ntfa_reader_hdl_rec_t *reader_list;
>>> SYSF_MBX mbx; /* priority q mbx b/w MDS & Library */
>>> struct ntfa_client_hdl_rec *next; /* next pointer for the
>>> list in ntfa_cb_t */
>>> + bool valid; /* handle is valid if it's known by NTF
>>> server, used for headless hydra */
>>> + SaVersionT version; /* the API version is being used by client,
>>> used for recover after headless */
>>> } ntfa_client_hdl_rec_t;
>>>
>>> /*
>>> * The NTFA control block is the master anchor structure for all NTFA
>>> * instantiations within a process.
>>> */
>>> +typedef enum {
>>> + NTFA_NTFSV_NONE = 0,
>>> + NTFA_NTFSV_DOWN,
>>> + NTFA_NTFSV_NO_ACTIVE,
>>> + NTFA_NTFSV_NEW_ACTIVE,
>>> + NTFA_NTFSV_UP
>>> +}ntfa_ntfsv_state_t;
>>> +
>>> typedef struct {
>>> pthread_mutex_t cb_lock; /* CB lock */
>>> ntfa_client_hdl_rec_t *client_list; /* NTFA client handle
>>> database */
>>> ntfa_reader_hdl_rec_t *reader_list;
>>> MDS_HDL mds_hdl; /* MDS handle */
>>> MDS_DEST ntfs_mds_dest; /* NTFS absolute/virtual address */
>>> - int ntfs_up; /* Indicate that MDS subscription
>>> - * is complete */
>>> +
>>> /* NTFS NTFA sync params */
>>> int ntfs_sync_awaited;
>>> NCS_SEL_OBJ ntfs_sync_sel;
>>> SaUint32T ntf_var_data_limit; /* max allowed
>>> variableDataSize */
>>> + /* NTF Server state */
>>> + ntfa_ntfsv_state_t ntfa_ntfsv_state;
>>> } ntfa_cb_t;
>>>
>>> /* ntfa_saf_api.c */
>>> @@ -149,7 +165,7 @@ extern void ntfsv_ntfa_evt_free(struct n
>>>
>>> /* ntfa_init.c */
>>> extern unsigned int ntfa_startup(void);
>>> -extern unsigned int ntfa_shutdown(void);
>>> +extern unsigned int ntfa_shutdown(bool forced);
>>>
>>> /* ntfa_hdl.c */
>>> extern SaAisErrorT ntfa_hdl_cbk_dispatch(ntfa_cb_t *,
>>> ntfa_client_hdl_rec_t *, SaDispatchFlagsT);
>>> @@ -159,6 +175,7 @@ extern ntfa_notification_hdl_rec_t *ntfa
>>> extern ntfa_filter_hdl_rec_t
>>> *ntfa_filter_hdl_rec_add(ntfa_client_hdl_rec_t **hdl_rec);
>>> extern void ntfa_hdl_list_del(ntfa_client_hdl_rec_t **);
>>> extern uint32_t ntfa_hdl_rec_del(ntfa_client_hdl_rec_t **,
>>> ntfa_client_hdl_rec_t *);
>>> +extern void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **,
>>> ntfa_client_hdl_rec_t *);
>>> extern uint32_t
>>> ntfa_notification_hdl_rec_del(ntfa_notification_hdl_rec_t **,
>>> ntfa_notification_hdl_rec_t *);
>>> extern uint32_t ntfa_filter_hdl_rec_del(ntfa_filter_hdl_rec_t **,
>>> ntfa_filter_hdl_rec_t *);
>>> extern bool ntfa_validate_ntfa_client_hdl(ntfa_cb_t *ntfa_cb,
>>> ntfa_client_hdl_rec_t *find_hdl_rec);
>>> @@ -166,11 +183,15 @@ extern bool ntfa_validate_ntfa_client_hd
>>> /* ntfa_util.c */
>>> extern ntfa_client_hdl_rec_t
>>> *ntfa_find_hdl_rec_by_client_id(ntfa_cb_t *ntfa_cb, uint32_t client_id);
>>> extern void ntfa_msg_destroy(ntfsv_msg_t *msg);
>>> -extern void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t
>>> *instance);
>>> -extern void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t
>>> +extern void ntfa_notification_destructor(ntfa_notification_hdl_rec_t
>>> *instance);
>>> +extern void ntfa_filter_destructor(ntfa_filter_hdl_rec_t
>>> *notificationFilterInstance);
>>> extern ntfa_reader_hdl_rec_t
>>> *ntfa_reader_hdl_rec_add(ntfa_client_hdl_rec_t **hdl_rec);
>>> extern uint32_t ntfa_reader_hdl_rec_del(ntfa_reader_hdl_rec_t **,
>>> ntfa_reader_hdl_rec_t *);
>>> extern void ntfa_add_to_async_cbk_msg_list(ntfsv_msg_t ** head,
>>> ntfsv_msg_t * new_node);
>>> extern uint32_t ntfa_ntfs_msg_proc(ntfa_cb_t *cb, ntfsv_msg_t
>>> *ntfsv_msg, MDS_SEND_PRIORITY_TYPE prio);
>>> +extern void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState);
>>> +extern SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
>>> + const ntfsv_filter_ptrs_t* pSrc);
>>> +extern SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t*
>>> filter_ptrs);
>>> #endif /* !NTFA_H */
>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>> b/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>> --- a/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>> @@ -34,11 +34,42 @@
>>> /* The main controle block */
>>> ntfa_cb_t ntfa_cb = {
>>> .cb_lock = PTHREAD_MUTEX_INITIALIZER,
>>> + .ntfa_ntfsv_state = NTFA_NTFSV_NONE,
>>> };
>>>
>>> /* list of subscriptions for this process */
>>> ntfa_subscriber_list_t *subscriberNoList = NULL;
>>>
>>> +/*
>>> + * @Brief: Determine the common "availability" of API, which depends
>>> on the
>>> + * NTF Server state
>>> + * @Param: None
>>> + * @Return: OK - If Server state is UP
>>> + * TRY_AGAIN - If Server state is No Active (temporary no
>>> active)
>>> + * BAD_HANDLE - If Server state is completely Down (no
>>> active at all).
>>> + * This error code will be returned to client
>>> calling API
>>> + */
>>> +static SaAisErrorT checkNtfServerState()
>>> +{
>>> + SaAisErrorT rc = SA_AIS_ERR_NOT_SUPPORTED;
>>> + /* Check NTF server availability */
>>> + if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_UP) {
>>> + rc = SA_AIS_OK;
>>> + } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_DOWN) {
>>> + TRACE("NTFS server is down");
>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>> [Praveen] 1. Function header says BAD_HANDLE for "down" but here
>> TRY_AGAIN is returned.
> [Minh]: I got similar comments from Lennart, I will correct the function
> description
>> 2. If server state is down, it means both ntfs are not up which means
>> headless cluster. But in case of normal cluster also application will
>> get TRY_AGAIN. Are you relying on the fact that in normal
>> cluster case payloads will be rebooted? if answer is yes then what if
>> reboot is delayed by changing the opensaf_reboot script which is
>> customizable?
> [Minh]: I don't get your question. No impact if payloads reboot
[Praveen] Actually I want draw attention to two problems here:
1) Normal cluster case: Both controllers are up in normal cluster
(attribute commented in immd.conf) and some ntf application is running
on the payload. Now if opensaf is stopped on the payload, application
will still get TRY_AGIAN. Now if a fresh controller is started and
payload is not started, the application API will be successful. But this
application should not be able to get a success for any API since its
payload is down.
2) Headless cluster case: Same case when with headless cluster
configured (attribute un-commented in immd.conf).
I think this may be the generic problem with all the service with two
tier architecture and a fix is needed. There should be a mechanism
though which agent should detect opensaf shutdown on local node and
clear all clients data.
>>
>>> + } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NO_ACTIVE) {
>>> + TRACE("NTFS server is unavailable");
>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>> + } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NONE) {
>>> + TRACE("No NTF server is detected, or API is called during
>>> headless");
>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>> + } else
>>> + TRACE("Not supported API call under NTF Server state (%u)",
>>> + ntfa_cb.ntfa_ntfsv_state);
>>> + return rc;
>>> +}
>>> +
>>> static SaAisErrorT checkNtfValueTypeRange(SaNtfValueTypeT type)
>>> {
>>> return (type < SA_NTF_VALUE_UINT8 || type >
>>> SA_NTF_VALUE_ARRAY)? SA_AIS_ERR_INVALID_PARAM
>>> @@ -871,7 +902,266 @@ SaAisErrorT getFilters(const SaNtfNotifi
>>> }
>>>
>>> /* end help functions */
>>> -
>>> +/****************************************************************************
>>>
>>> + Name : reinitializeClient
>>> +
>>> + Description : This routine sends initialize messages to NTF
>>> Server in
>>> + order to obtain new client id
>>> +
>>> + Arguments : client_hdl* [IN]: client handle
>>> +
>>> + Return Values : SA_AIS_OK if success, others are failure
>>> +
>>> + Notes : None
>>> +******************************************************************************/
>>>
>>> +SaAisErrorT reinitializeClient(ntfa_client_hdl_rec_t* client_hdl) {
>>> + uint32_t mds_rc;
>>> + SaAisErrorT rc = SA_AIS_OK;
>>> + ntfsv_msg_t i_msg, *o_msg = NULL;
>>> +
>>> + TRACE_ENTER();
>>> +
>>> + memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>>> + i_msg.type = NTFSV_NTFA_API_MSG;
>>> + i_msg.info.api_info.type = NTFSV_INITIALIZE_REQ;
>>> + i_msg.info.api_info.param.init.version = client_hdl->version;
>>> +
>>> + mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>>> NTFS_WAIT_TIME);
>>> + switch (mds_rc) {
>>> + case NCSCC_RC_SUCCESS:
>>> + break;
>>> + case NCSCC_RC_REQ_TIMOUT:
>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>> + goto done;
>>> + default:
>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>> + goto done;
>>> + }
>>> +
>>> + osafassert(o_msg != NULL);
>>> + /* Check response msg: type, rc */
>>> + if (o_msg->info.api_resp_info.type != NTFSV_INITIALIZE_RSP) {
>>> + TRACE("info.api_resp_info.type:%u",
>>> o_msg->info.api_resp_info.type);
>>> + rc = SA_AIS_ERR_LIBRARY;
>>> + goto done;
>>> + }
>>> + if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>>> + TRACE("info.api_resp_info.rc:%u",
>>> o_msg->info.api_resp_info.rc);
>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>> + goto done;
>>> + }
>>> +
>>> + client_hdl->ntfs_client_id =
>>> o_msg->info.api_resp_info.param.init_rsp.client_id;
>>> + TRACE("Successfully recover client_id, new client_id:%u",
>>> + client_hdl->ntfs_client_id);
>>> +
>>> +done:
>>> + if (o_msg)
>>> + ntfa_msg_destroy(o_msg);
>>> + TRACE_LEAVE();
>>> + return rc;
>>> +}
>>> +/****************************************************************************
>>>
>>> + Name : recoverReader
>>> +
>>> + Description : This routine sends read_initialize messages to NTF
>>> Server in
>>> + order to reintroduce this reader to NTF Server
>>> +
>>> + Arguments : client_hdl* [IN]: client handle
>>> + reader_hdl* [IN]: reader handle
>>> +
>>> + Return Values : SA_AIS_OK if success, others are failure
>>> +
>>> + Notes : None
>>> +******************************************************************************/
>>>
>>> +SaAisErrorT recoverReader(ntfa_client_hdl_rec_t* client_hdl,
>>> ntfa_reader_hdl_rec_t* reader_hdl) {
>>> + uint32_t mds_rc;
>>> + SaAisErrorT rc = SA_AIS_OK;
>>> + ntfsv_msg_t i_msg, *o_msg = NULL;
>>> + ntfsv_reader_init_req_2_t *send_param;
>>> +
>>> + TRACE_ENTER();
>>> +
>>> + /* Make sure the filters have not been deleted */
>>> + if (reader_hdl->filters.alarm_filter == NULL &&
>>> + reader_hdl->filters.att_ch_filter == NULL &&
>>> + reader_hdl->filters.obj_cr_del_filter == NULL &&
>>> + reader_hdl->filters.sec_al_filter == NULL &&
>>> + reader_hdl->filters.sta_ch_filter == NULL)
>>> + return SA_AIS_ERR_BAD_HANDLE;
>>> +
>>> + memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>>> + i_msg.type = NTFSV_NTFA_API_MSG;
>>> + i_msg.info.api_info.type = NTFSV_READER_INITIALIZE_REQ_2;
>>> + send_param = &i_msg.info.api_info.param.reader_init_2;
>>> + send_param->head.client_id = client_hdl->ntfs_client_id;
>>> + send_param->head.searchCriteria = reader_hdl->searchCriteria;
>>> + send_param->f_rec = reader_hdl->filters;
>>> +
>>> + mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>>> NTFS_WAIT_TIME);
>>> +
>>> + switch (mds_rc) {
>>> + case NCSCC_RC_SUCCESS:
>>> + break;
>>> + case NCSCC_RC_REQ_TIMOUT:
>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>> + goto done;
>>> + default:
>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>> + goto done;
>>> + }
>>> +
>>> + osafassert(o_msg != NULL);
>>> + if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>>> + TRACE("o_msg->info.api_resp_info.rc:%u",
>>> o_msg->info.api_resp_info.rc);
>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>> + goto done;
>>> + }
>>> +
>>> + if (o_msg->info.api_resp_info.type !=
>>> NTFSV_READER_INITIALIZE_RSP) {
>>> + TRACE("msg type (%d) failed",
>>> (int)o_msg->info.api_resp_info.type);
>>> + rc = SA_AIS_ERR_LIBRARY;
>>> + goto done;
>>> + }
>>> + /* Update reader_id since it may be changed */
>>> + reader_hdl->reader_id =
>>> o_msg->info.api_resp_info.param.reader_init_rsp.readerId;
>>> +
>>> + TRACE("Recover reader successfully");
>>> +done:
>>> + if (o_msg)
>>> + ntfa_msg_destroy(o_msg);
>>> + TRACE_LEAVE();
>>> + return rc;
>>> +}
>>> +/****************************************************************************
>>>
>>> + Name : recoverSubscriber
>>> +
>>> + Description : This routine sends subscribe messages to NTF
>>> Server in
>>> + order to reintroduce this subscriber to NTF Server
>>> +
>>> + Arguments : client_hdl* [IN]: client handle
>>> + subscriber_hdl* [IN]: subscriber handle
>>> +
>>> + Return Values : SA_AIS_OK if success, others are failure
>>> +
>>> + Notes : None
>>> +******************************************************************************/
>>>
>>> +SaAisErrorT recoverSubscriber(ntfa_client_hdl_rec_t* client_hdl,
>>> + ntfa_subscriber_list_t*
>>> subscriber_hdl) {
>>> + uint32_t mds_rc;
>>> + SaAisErrorT rc = SA_AIS_OK;
>>> + ntfsv_msg_t i_msg, *o_msg = NULL;
>>> + ntfsv_subscribe_req_t *send_param;
>>> +
>>> + TRACE_ENTER();
>>> +
>>> + /* Make sure the filters have not been deleted */
>>> + if (subscriber_hdl->filters.alarm_filter == NULL &&
>>> + subscriber_hdl->filters.att_ch_filter == NULL &&
>>> + subscriber_hdl->filters.obj_cr_del_filter == NULL &&
>>> + subscriber_hdl->filters.sec_al_filter == NULL &&
>>> + subscriber_hdl->filters.sta_ch_filter == NULL)
>>> + return SA_AIS_ERR_BAD_HANDLE;
>>> +
>>> + memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>>> + i_msg.type = NTFSV_NTFA_API_MSG;
>>> + i_msg.info.api_info.type = NTFSV_SUBSCRIBE_REQ;
>>> + send_param = &i_msg.info.api_info.param.subscribe;
>>> +
>>> + send_param->client_id = client_hdl->ntfs_client_id;
>>> + send_param->subscriptionId =
>>> subscriber_hdl->subscriberListSubscriptionId;
>>> + send_param->f_rec = subscriber_hdl->filters;
>>> +
>>> + mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>>> NTFS_WAIT_TIME);
>>> +
>>> + switch (mds_rc) {
>>> + case NCSCC_RC_SUCCESS:
>>> + break;
>>> + case NCSCC_RC_REQ_TIMOUT:
>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>> + goto done;
>>> + default:
>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>> + goto done;
>>> + }
>>> +
>>> + osafassert(o_msg != NULL);
>>> +
>>> + if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>>> + TRACE("o_msg->info.api_resp_info.rc:%u",
>>> o_msg->info.api_resp_info.rc);
>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>> + goto done;
>>> + }
>>> +
>>> + if (o_msg->info.api_resp_info.type != NTFSV_SUBSCRIBE_RSP) {
>>> + TRACE("msg type (%d) failed",
>>> (int)o_msg->info.api_resp_info.type);
>>> + rc = SA_AIS_ERR_LIBRARY;
>>> + goto done;
>>> + }
>>> + TRACE("Recover subscriber successfully");
>>> +done:
>>> + if (o_msg)
>>> + ntfa_msg_destroy(o_msg);
>>> +
>>> + TRACE_LEAVE();
>>> + return rc;
>>> +}
>>> +/****************************************************************************
>>>
>>> + Name : recoverClient
>>> +
>>> + Description : In NTF-A.03.01, section 2.1.2. A client can be
>>> producer,
>>> + subscriber, and reader at the same time.
>>> + This routine will recovery:
>>> + (1) client_id
>>> + (2) Then it will continue the recovery if this client also
>>> + has instance of subscriber or reader.
>>> + Arguments : client_hdl* [IN]: client handle
>>> +
>>> + Return Values : SA_AIS_OK if success, others are failure
>>> +
>>> + Notes : None
>>> +******************************************************************************/
>>>
>>> +SaAisErrorT recoverClient(ntfa_client_hdl_rec_t *client_hdl) {
>>> +
>>> + TRACE_ENTER();
>>> + SaAisErrorT rc = SA_AIS_OK;
>>> +
>>> + osafassert(client_hdl);
>>> +
>>> + if ((rc = reinitializeClient(client_hdl)) == SA_AIS_OK) {
>>> + /* Restore reader */
>>> + ntfa_reader_hdl_rec_t* reader_hdl = client_hdl->reader_list;
>>> + while (reader_hdl != NULL && rc == SA_AIS_OK) {
>>> + rc = recoverReader(client_hdl, reader_hdl);
>>> + reader_hdl = reader_hdl->next;
>>> + }
>>> + if (rc != SA_AIS_OK) {
>>> + TRACE("Failed to restore reader (readerId:%d)",
>>> + reader_hdl->reader_id);
>>> + goto done;
>>> + }
>>> + /* Restore subscriber */
>>> + ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
>>> + while (subscriber_hdl != NULL && rc == SA_AIS_OK) {
>>> + if (client_hdl->local_hdl ==
>>> subscriber_hdl->subscriberListNtfHandle)
>>> + rc = recoverSubscriber(client_hdl, subscriber_hdl);
>>> + subscriber_hdl = subscriber_hdl->next;
>>> + }
>>> + if (rc != SA_AIS_OK) {
>>> + TRACE("Failed to restore subscriber (subscriptionId:%d)",
>>> + subscriber_hdl->subscriberListSubscriptionId);
>>> + goto done;
>>> + }
>>> + client_hdl->valid = true;
>>> + } else {
>>> + TRACE("Failed to restore client (id:%d)",
>>> client_hdl->ntfs_client_id);
>>> + goto done;
>>> + }
>>> +
>>> +done:
>>> + TRACE_LEAVE();
>>> + return rc;
>>> +}
>>> /***************************************************************************
>>>
>>> * 8.4.1
>>> *
>>> @@ -923,15 +1213,14 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>> version->releaseCode = NTF_RELEASE_CODE;
>>> version->majorVersion = NTF_MAJOR_VERSION;
>>> version->minorVersion = NTF_MINOR_VERSION;
>>> - ntfa_shutdown();
>>> + ntfa_shutdown(false);
>>> rc = SA_AIS_ERR_VERSION;
>>> goto done;
>>> }
>>>
>>> - if (!ntfa_cb.ntfs_up) {
>>> - ntfa_shutdown();
>>> - TRACE("NTFS server is down");
>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>> + /* Check NTF server availability */
>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>>> + ntfa_shutdown(false);
>>> goto done;
>>> }
>>>
>>> @@ -977,7 +1266,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>> rc = SA_AIS_ERR_NO_MEMORY;
>>> goto err;
>>> }
>>> -
>>> + ntfa_hdl_rec->version = *version;
>>> /* pass the handle value to the appl */
>>> if (SA_AIS_OK == rc)
>>> *ntfHandle = ntfa_hdl_rec->local_hdl;
>>> @@ -990,7 +1279,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>>
>>> if (rc != SA_AIS_OK) {
>>> TRACE_2("NTFA INIT FAILED\n");
>>> - ntfa_shutdown();
>>> + ntfa_shutdown(false);
>>> }
>>>
>>> done:
>>> @@ -1101,6 +1390,28 @@ SaAisErrorT saNtfDispatch(SaNtfHandleT n
>>> goto done;
>>> }
>>>
>>> + /* Need to check NTF server availability here
>>> + * Because if the notificationCallback just comes after MDS_DOWN
>>> + * it will go to recovery then result in failure due to no director
>>> + */
>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>>> + ncshm_give_hdl(ntfHandle);
>>> + goto done;
>>> + }
>>> +
>>> + if (!hdl_rec->valid) {
>>> + /* recovery */
>>> + if ((rc = recoverClient(hdl_rec)) != SA_AIS_OK) {
>>> + if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>> + ncshm_give_hdl(ntfHandle);
>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, hdl_rec);
>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_shutdown(false);
>>> + goto done;
>>> + }
>>> + }
>>> + }
>>> if ((rc = ntfa_hdl_cbk_dispatch(&ntfa_cb, hdl_rec,
>>> dispatchFlags)) != SA_AIS_OK)
>>> TRACE("NTFA_DISPATCH_FAILURE");
>>>
>>> @@ -1145,6 +1456,13 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>>
>>> TRACE_ENTER();
>>>
>>> + /* Check NTF server availability */
>>> + if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NO_ACTIVE) {
>>> + TRACE("NTFS server is temporarily unavailable");
>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>> + goto done;
>>> + }
>>> +
>>> /* retrieve hdl rec */
>>> hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA, ntfHandle);
>>> if (hdl_rec == NULL) {
>>> @@ -1152,43 +1470,37 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>> rc = SA_AIS_ERR_BAD_HANDLE;
>>> goto done;
>>> }
>>> -
>>> - /* Check Whether NTFS is up or not */
>>> - if (!ntfa_cb.ntfs_up) {
>>> - TRACE("NTFS down");
>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>> - goto done_give_hdl;
>>> + if (hdl_rec->valid) {
>>> + /** populate & send the finalize message
>>> + ** and make sure the finalize from the server
>>> + ** end returned before deleting the local records.
>>> + **/
>>> + memset(&msg, 0, sizeof(ntfsv_msg_t));
>>> + msg.type = NTFSV_NTFA_API_MSG;
>>> + msg.info.api_info.type = NTFSV_FINALIZE_REQ;
>>> + msg.info.api_info.param.finalize.client_id =
>>> hdl_rec->ntfs_client_id;
>>> +
>>> + mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>>> NTFS_WAIT_TIME);
>>> + switch (mds_rc) {
>>> + case NCSCC_RC_SUCCESS:
>>> + break;
>>> + case NCSCC_RC_REQ_TIMOUT:
>>> + rc = SA_AIS_ERR_TIMEOUT;
>>> + TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>> + goto done_give_hdl;
>>> + default:
>>> + TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>> + rc = SA_AIS_ERR_NO_RESOURCES;
>>> + goto done_give_hdl;
>>> + }
>>> +
>>> + if (o_msg != NULL) {
>>> + rc = o_msg->info.api_resp_info.rc;
>>> + ntfa_msg_destroy(o_msg);
>>> + } else
>>> + rc = SA_AIS_ERR_NO_RESOURCES;
>>> }
>>>
>>> - /** populate & send the finalize message
>>> - ** and make sure the finalize from the server
>>> - ** end returned before deleting the local records.
>>> - **/
>>> - memset(&msg, 0, sizeof(ntfsv_msg_t));
>>> - msg.type = NTFSV_NTFA_API_MSG;
>>> - msg.info.api_info.type = NTFSV_FINALIZE_REQ;
>>> - msg.info.api_info.param.finalize.client_id =
>>> hdl_rec->ntfs_client_id;
>>> -
>>> - mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>>> NTFS_WAIT_TIME);
>>> - switch (mds_rc) {
>>> - case NCSCC_RC_SUCCESS:
>>> - break;
>>> - case NCSCC_RC_REQ_TIMOUT:
>>> - rc = SA_AIS_ERR_TIMEOUT;
>>> - TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>> - goto done_give_hdl;
>>> - default:
>>> - TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>> - rc = SA_AIS_ERR_NO_RESOURCES;
>>> - goto done_give_hdl;
>>> - }
>>> -
>>> - if (o_msg != NULL) {
>>> - rc = o_msg->info.api_resp_info.rc;
>>> - ntfa_msg_destroy(o_msg);
>>> - } else
>>> - rc = SA_AIS_ERR_NO_RESOURCES;
>>> -
>>> if (rc == SA_AIS_OK) {
>>> /** delete the hdl rec
>>> ** including all resources allocated by client if MDS send is
>>> @@ -1205,7 +1517,7 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>> ncshm_give_hdl(ntfHandle);
>>>
>>> if (rc == SA_AIS_OK) {
>>> - rc = ntfa_shutdown();
>>> + rc = ntfa_shutdown(false);
>>> if (rc != NCSCC_RC_SUCCESS)
>>> TRACE_1("ntfa_shutdown failed");
>>> }
>>> @@ -1349,7 +1661,7 @@ SaAisErrorT saNtfNotificationFree(SaNtfN
>>> }
>>>
>>> /* free the resources allocated by
>>> saNtf<ntfType>NotificationAllocate */
>>> - ntfa_hdl_rec_destructor(notification_hdl_rec);
>>> + ntfa_notification_destructor(notification_hdl_rec);
>>>
>>> /** Delete the resources related to the notificationHandle &
>>> * remove reference in the client.
>>> @@ -1419,6 +1731,24 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>>> }
>>> osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>
>>> + /* Check NTF server availability */
>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>> + goto done_give_hdls;
>>> +
>>> + /* Recover if this is invalid handle */
>>> + if (!client_rec->valid) {
>>> + if ((rc = recoverClient(client_rec)) != SA_AIS_OK) {
>>> + ncshm_give_hdl(client_handle);
>>> + ncshm_give_hdl(notificationHandle);
>>> + if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_rec);
>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_shutdown(false);
>>> + }
>>> + goto err_free;
>>> + }
>>> + }
>>> /**
>>> ** Populate a sync MDS message
>>> **/
>>> @@ -1481,13 +1811,6 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>>> goto done_give_hdls;
>>> }
>>>
>>> - /* Check whether NTFS is up or not */
>>> - if (!ntfa_cb.ntfs_up) {
>>> - TRACE("NTFS down");
>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>> - goto done_give_hdls;
>>> - }
>>> -
>>> /* Send a sync MDS message to obtain a notification id */
>>> mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>> switch (mds_rc) {
>>> @@ -1596,7 +1919,8 @@ static SaNtfHandleT ntfHandleGet(SaNtfS
>>> return ntfHandle;
>>> }
>>>
>>> -static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
>>> SaNtfSubscriptionIdT subscriptionId)
>>> +static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
>>> SaNtfSubscriptionIdT subscriptionId,
>>> + ntfsv_filter_ptrs_t filters)
>>> {
>>> SaAisErrorT rc = SA_AIS_OK;
>>> ntfa_subscriber_list_t* ntfSubscriberList;
>>> @@ -1610,7 +1934,9 @@ static SaAisErrorT subscriptionListAdd(S
>>> /* Add ntfHandle and subscriptionId into list */
>>> ntfSubscriberList->subscriberListNtfHandle = ntfHandle;
>>> ntfSubscriberList->subscriberListSubscriptionId = subscriptionId;
>>> -
>>> + memset(&ntfSubscriberList->filters, 0,
>>> sizeof(ntfsv_filter_ptrs_t));
>>> + ntfa_copy_ntf_filter_ptrs(&ntfSubscriberList->filters, &filters);
>>> +
>>> osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>> if (NULL == subscriberNoList) {
>>> subscriberNoList = ntfSubscriberList;
>>> @@ -1648,6 +1974,7 @@ static void subscriberListItemRemove(SaN
>>> subscriberNoList = NULL;
>>> }
>>> TRACE_1("REMOVE: listPtr->SubscriptionId %d",
>>> listPtr->subscriberListSubscriptionId);
>>> + ntfa_del_ntf_filter_ptrs(&listPtr->filters);
>>> free(listPtr);
>>> osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>> }
>>> @@ -1666,20 +1993,33 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>> ntfsv_msg_t msg, *o_msg = NULL;
>>> ntfsv_subscribe_req_t *send_param;
>>> uint32_t timeout = NTFS_WAIT_TIME;
>>> -
>>> + bool recovery_failed = false;
>>> TRACE_ENTER();
>>> +
>>> rc = getFilters(notificationFilterHandles, &filters,
>>> &ntfHandle, &client_hdl_rec);
>>> if (rc != SA_AIS_OK) {
>>> TRACE("getFilters failed");
>>> goto done;
>>> }
>>>
>>> + /* Check NTF server availability */
>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>> + goto done;
>>> +
>>> + /* recovery */
>>> + if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
>>> + if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>> + recovery_failed = true;
>>> + goto done;
>>> + }
>>> + }
>>> +
>>> tmpHandle = ntfHandleGet(subscriptionId);
>>> if (tmpHandle != 0) {
>>> rc = SA_AIS_ERR_EXIST;
>>> goto done;
>>> }
>>> - rc = subscriptionListAdd(ntfHandle, subscriptionId);
>>> + rc = subscriptionListAdd(ntfHandle, subscriptionId, filters);
>>> if (rc != SA_AIS_OK) {
>>> goto done;
>>> }
>>> @@ -1693,29 +2033,24 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>> send_param->client_id = client_hdl_rec->ntfs_client_id;
>>> send_param->subscriptionId = subscriptionId;
>>> send_param->f_rec = filters;
>>> - /* Check whether NTFS is up or not */
>>> - if (ntfa_cb.ntfs_up) {
>>> - uint32_t rv;
>>> - /* Send a sync MDS message to obtain a log stream id */
>>> - rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>> - if (rv == NCSCC_RC_SUCCESS) {
>>> - osafassert(o_msg != NULL);
>>> - if (SA_AIS_OK == o_msg->info.api_resp_info.rc) {
>>> - TRACE_1("subscriptionId from server %u",
>>> - o_msg->info.api_resp_info.param.subscribe_rsp.subscriptionId);
>>> - } else {
>>> - rc = o_msg->info.api_resp_info.rc;
>>> - TRACE("Bad return status!!! rc = %d", rc);
>>> - }
>>> +
>>> + uint32_t rv;
>>> + /* Send a sync MDS message to obtain a log stream id */
>>> + rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>> + if (rv == NCSCC_RC_SUCCESS) {
>>> + osafassert(o_msg != NULL);
>>> + if (SA_AIS_OK == o_msg->info.api_resp_info.rc) {
>>> + TRACE_1("subscriptionId from server %u",
>>> + o_msg->info.api_resp_info.param.subscribe_rsp.subscriptionId);
>>> } else {
>>> - if(rv == NCSCC_RC_INVALID_INPUT)
>>> - rc = SA_AIS_ERR_INVALID_PARAM;
>>> - else
>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>> + rc = o_msg->info.api_resp_info.rc;
>>> + TRACE("Bad return status!!! rc = %d", rc);
>>> }
>>> } else {
>>> - TRACE_1("NTFS down");
>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>> + if(rv == NCSCC_RC_INVALID_INPUT)
>>> + rc = SA_AIS_ERR_INVALID_PARAM;
>>> + else
>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>> }
>>>
>>> if (rc != SA_AIS_OK) {
>>> @@ -1724,6 +2059,7 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>> if (o_msg)
>>> ntfa_msg_destroy(o_msg);
>>> done:
>>> +
>>> ncshm_give_hdl(ntfHandle);
>>> if (notificationFilterHandles) {
>>> if (notificationFilterHandles->attributeChangeFilterHandle)
>>> @@ -1737,6 +2073,13 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>> if (notificationFilterHandles->alarmFilterHandle)
>>> ncshm_give_hdl(notificationFilterHandles->alarmFilterHandle);
>>> }
>>> + if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_shutdown(false);
>>> + }
>>> +
>>> TRACE_LEAVE();
>>> return rc;
>>> }
>>> @@ -1995,6 +2338,7 @@ SaAisErrorT saNtfSecurityAlarmNotificati
>>> rc = SA_AIS_ERR_INVALID_PARAM;
>>> goto done;
>>> }
>>> +
>>> /* retrieve hdl rec */
>>> hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA, ntfHandle);
>>> if (hdl_rec == NULL) {
>>> @@ -2058,6 +2402,7 @@ SaAisErrorT saNtfPtrValAllocate(SaNtfNot
>>> rc = SA_AIS_ERR_INVALID_PARAM;
>>> goto done;
>>> }
>>> +
>>> notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>>> notificationHandle);
>>> if (notification_hdl_rec == NULL) {
>>> TRACE("ncshm_take_hdl notificationHandle failed");
>>> @@ -2098,6 +2443,7 @@ SaAisErrorT saNtfArrayValAllocate(SaNtfN
>>> rc = SA_AIS_ERR_INVALID_PARAM;
>>> goto done;
>>> }
>>> +
>>> notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>>> notificationHandle);
>>> if (notification_hdl_rec == NULL) {
>>> TRACE("ncshm_take_hdl notificationHandle failed");
>>> @@ -2228,6 +2574,7 @@ SaAisErrorT saNtfArrayValGet(SaNtfNotifi
>>> rc = SA_AIS_ERR_INVALID_PARAM;
>>> goto done;
>>> }
>>> +
>>> notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>>> notificationHandle);
>>> if (notification_hdl_rec == NULL) {
>>> TRACE("ncshm_take_hdl notificationHandle failed");
>>> @@ -2695,7 +3042,7 @@ SaAisErrorT saNtfNotificationFilterFree(
>>> }
>>>
>>> /* free the resources allocated by saNtf<ntfType>FilterAllocate */
>>> - ntfa_filter_hdl_rec_destructor(filter_hdl_rec);
>>> + ntfa_filter_destructor(filter_hdl_rec);
>>>
>>> /** Delete the resources related to the notificationFilterHandle &
>>> * remove reference in the client.
>>> @@ -2729,7 +3076,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>> TRACE_ENTER();
>>> SaAisErrorT rc = SA_AIS_ERR_NOT_EXIST;
>>> SaNtfHandleT ntfHandle;
>>> -
>>> ntfa_client_hdl_rec_t *client_hdl_rec;
>>>
>>> ntfsv_msg_t msg, *o_msg = NULL;
>>> @@ -2737,7 +3083,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>> ntfsv_unsubscribe_req_t *send_param;
>>> uint32_t timeout = NTFS_WAIT_TIME;
>>>
>>> -
>>> ntfHandle = ntfHandleGet(subscriptionId);
>>> if (ntfHandle == 0) {
>>> TRACE_1("ntfHandleGet failed, subscription not exist");
>>> @@ -2753,9 +3098,16 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>> goto done;
>>> }
>>>
>>> - /**
>>> - ** Populate a sync MDS message
>>> - **/
>>> + /* Check NTF server availability */
>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>> [Praveen] Here in both unavailability and down case, API will return
>> without clean up of subscriber information. But README in patch 3 tals
>> differently for the two cases.
>> I think handling like saNtfFinalize() way needs to be done here.
> [Minh]: I will have to check the history commit, to see if any reason
> that agent doesn't silently release subscriber's resource here as
> described in README. But it looks to me a bug, can I get back to you later?
[Praveen] This needs to be corrected otherwise the same application will
not be able to reuse old subscription_id. It will get ERR_EXIST.
>>
>>> + ncshm_give_hdl(ntfHandle);
>>> + goto done;
>>> + }
>>> +
>>> + if (client_hdl_rec->valid) {
>>> + /**
>>> + ** Populate a sync MDS message
>>> + **/
>>> memset(&msg, 0, sizeof(ntfsv_msg_t));
>>> msg.type = NTFSV_NTFA_API_MSG;
>>> msg.info.api_info.type = NTFSV_UNSUBSCRIBE_REQ;
>>> @@ -2764,13 +3116,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>> send_param->client_id = client_hdl_rec->ntfs_client_id;
>>> send_param->subscriptionId = subscriptionId;
>>>
>>> - /* Check whether NTFS is up or not */
>>> - if (!ntfa_cb.ntfs_up) {
>>> - TRACE_1("NTFS down");
>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>> - goto done_give_hdl;
>>> - }
>>> -
>>> /* Send a sync MDS message to obtain a log stream id */
>>> rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>> if (rc != NCSCC_RC_SUCCESS) {
>>> @@ -2785,6 +3130,7 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>> TRACE_1("Bad return status! rc = %d", rc);
>>> goto done_give_hdl;
>>> }
>>> + }
>>> subscriberListItemRemove(subscriptionId);
>>>
>>> /*Remove msg for subscriptionId from mailbox*/
>>> @@ -2815,6 +3161,19 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>> done_give_hdl:
>>> if (o_msg)
>>> ntfa_msg_destroy(o_msg);
>>> +
>>> + if (!client_hdl_rec->valid) {
>>> + if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>> + if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>> + ncshm_give_hdl(ntfHandle);
>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_shutdown(false);
>>> + goto done;
>>> + }
>>> + }
>>> + }
>>> ncshm_give_hdl(ntfHandle);
>>> done:
>>> TRACE_LEAVE();
>>> @@ -2838,6 +3197,7 @@ SaAisErrorT saNtfNotificationReadInitial
>>> ntfsv_reader_init_req_2_t *send_param;
>>> uint32_t timeout = NTFS_WAIT_TIME;
>>>
>>> + bool recovery_failed = false;
>>> TRACE_ENTER();
>>> if (notificationFilterHandles == NULL || readHandle == NULL) {
>>> rc = SA_AIS_ERR_INVALID_PARAM;
>>> @@ -2872,6 +3232,18 @@ SaAisErrorT saNtfNotificationReadInitial
>>> goto done_give_client_hdl;
>>> }
>>>
>>> + /* Check NTF server availability */
>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>> + goto done_give_client_hdl;
>>> +
>>> + /* recovery */
>>> + if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
>>> + if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>> + recovery_failed = true;
>>> + goto done_give_client_hdl;
>>> + }
>>> + }
>>> +
>>> /**
>>> ** Populate a sync MDS message
>>> **/
>>> @@ -2884,13 +3256,6 @@ SaAisErrorT saNtfNotificationReadInitial
>>> send_param->head.searchCriteria = searchCriteria;
>>> send_param->f_rec = filters;
>>>
>>> - /* Check whether NTFS is up or not */
>>> - if (!ntfa_cb.ntfs_up) {
>>> - TRACE("NTFS down");
>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>> - goto done_give_client_hdl;
>>> - }
>>> -
>>> /* Send a sync MDS message to obtain a log stream id */
>>> rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>> if (rc != NCSCC_RC_SUCCESS) {
>>> @@ -2925,12 +3290,16 @@ SaAisErrorT saNtfNotificationReadInitial
>>> reader_hdl_rec->ntfHandle = ntfHandle;
>>> /* Store the readerId returned from server */
>>> reader_hdl_rec->reader_id =
>>> o_msg->info.api_resp_info.param.reader_init_rsp.readerId;
>>> + memset(&reader_hdl_rec->filters, 0, sizeof(ntfsv_filter_ptrs_t));
>>> + ntfa_copy_ntf_filter_ptrs(&reader_hdl_rec->filters, &filters);
>>> + reader_hdl_rec->searchCriteria = searchCriteria;
>>> /** UnLock ntfa_CB **/
>>> osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>
>>> done_give_client_hdl:
>>> if (o_msg)
>>> ntfa_msg_destroy(o_msg);
>>> +
>>> if (client_hdl_rec)
>>> ncshm_give_hdl(client_hdl_rec->local_hdl);
>>> if (notificationFilterHandles) {
>>> @@ -2947,6 +3316,12 @@ done_give_client_hdl:
>>> }
>>>
>>> ncshm_give_hdl(notificationFilterHandles->alarmFilterHandle);
>>> + if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_shutdown(false);
>>> + }
>>> done:
>>> TRACE_LEAVE();
>>> return rc;
>>> @@ -2983,42 +3358,46 @@ SaAisErrorT saNtfNotificationReadFinaliz
>>> }
>>> TRACE_1("reader_hdl_rec = %u", reader_hdl_rec->reader_hdl);
>>>
>>> - /**
>>> - ** Populate a sync MDS message
>>> - **/
>>> - memset(&msg, 0, sizeof(ntfsv_msg_t));
>>> - msg.type = NTFSV_NTFA_API_MSG;
>>> - msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
>>> - send_param = &msg.info.api_info.param.reader_finalize;
>>> - send_param->client_id = client_hdl_rec->ntfs_client_id;
>>> - send_param->readerId = reader_hdl_rec->reader_id;
>>> -
>>> - /* Check whether NTFS is up or not */
>>> - if (!ntfa_cb.ntfs_up) {
>>> - TRACE("NTFS down");
>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>> - goto done_give_hdls;
>>> + /* Check NTF server availability */
>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>> [Praveen] Same comment here as in Unsubscribe().
>> [Minh] As above in Unsubscribed()
>>> + ncshm_give_hdl(client_hdl_rec->local_hdl);
>>> + ncshm_give_hdl(readhandle);
>>> + goto done;
>>> }
>>>
>>> - /* Send a sync MDS message */
>>> - rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>> - if (rc != NCSCC_RC_SUCCESS) {
>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>> - goto done_give_hdls;
>>> + if (client_hdl_rec->valid) {
>>> + /**
>>> + ** Populate a sync MDS message
>>> + **/
>>> + memset(&msg, 0, sizeof(ntfsv_msg_t));
>>> + msg.type = NTFSV_NTFA_API_MSG;
>>> + msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
>>> + send_param = &msg.info.api_info.param.reader_finalize;
>>> + send_param->client_id = client_hdl_rec->ntfs_client_id;
>>> + send_param->readerId = reader_hdl_rec->reader_id;
>>> +
>>> +
>>> + /* Send a sync MDS message */
>>> + rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>> + if (rc != NCSCC_RC_SUCCESS) {
>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>> + goto done_give_hdls;
>>> + }
>>> +
>>> + osafassert(o_msg != NULL);
>>> + if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
>>> + rc = o_msg->info.api_resp_info.rc;
>>> + TRACE("Bad return status!!! rc = %d", rc);
>>> + goto done_give_hdls;
>>> + }
>>> +
>>> + if (o_msg->info.api_resp_info.type !=
>>> NTFSV_READER_FINALIZE_RSP) {
>>> + TRACE("msg type (%d) failed",
>>> (int)o_msg->info.api_resp_info.type);
>>> + rc = SA_AIS_ERR_LIBRARY;
>>> + goto done_give_hdls;
>>> + }
>>> }
>>>
>>> - osafassert(o_msg != NULL);
>>> - if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
>>> - rc = o_msg->info.api_resp_info.rc;
>>> - TRACE("Bad return status!!! rc = %d", rc);
>>> - goto done_give_hdls;
>>> - }
>>> -
>>> - if (o_msg->info.api_resp_info.type != NTFSV_READER_FINALIZE_RSP) {
>>> - TRACE("msg type (%d) failed",
>>> (int)o_msg->info.api_resp_info.type);
>>> - rc = SA_AIS_ERR_LIBRARY;
>>> - goto done_give_hdls;
>>> - }
>>> osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>> oas_rc = ntfa_reader_hdl_rec_del(&client_hdl_rec->reader_list,
>>> reader_hdl_rec);
>>> if (oas_rc != NCSCC_RC_SUCCESS) {
>>> @@ -3029,6 +3408,20 @@ SaAisErrorT saNtfNotificationReadFinaliz
>>> done_give_hdls:
>>> if (o_msg)
>>> ntfa_msg_destroy(o_msg);
>>> +
>>> + if (!client_hdl_rec->valid) {
>>> + if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>> + if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>> + ncshm_give_hdl(client_hdl_rec->local_hdl);
>>> + ncshm_give_hdl(readhandle);
>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_shutdown(false);
>>> + goto done;
>>> + }
>>> + }
>>> + }
>>> ncshm_give_hdl(client_hdl_rec->local_hdl);
>>> done_give_read_hdl:
>>> ncshm_give_hdl(readhandle);
>>> @@ -3082,6 +3475,23 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>>> }
>>> TRACE_1("reader_hdl_rec = %u", reader_hdl_rec->reader_hdl);
>>>
>>> + /* Check NTF server availability */
>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>> + goto done_give_hdls;
>>> +
>>> + if (!client_hdl_rec->valid) {
>>> + if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>> + ncshm_give_hdl(client_hdl_rec->local_hdl);
>>> + ncshm_give_hdl(readHandle);
>>> + if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>> + ntfa_shutdown(false);
>>> + }
>>> + goto done;
>>> + }
>>> + }
>>> /**
>>> ** Populate a sync MDS message
>>> **/
>>> @@ -3092,12 +3502,6 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>>> send_param->client_id = client_hdl_rec->ntfs_client_id;
>>> send_param->readerId = reader_hdl_rec->reader_id;
>>> send_param->searchDirection = searchDirection;
>>> - /* Check whether NTFS is up or not */
>>> - if (!ntfa_cb.ntfs_up) {
>>> - TRACE("NTFS down");
>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>> - goto done_give_hdls;
>>> - }
>>>
>>> do {
>>> /* Send a sync MDS message */
>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>> b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>> --- a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>> @@ -369,7 +369,8 @@ static uint32_t ntfa_mds_svc_evt(struct
>>> TRACE("NTFS down");
>>> pthread_mutex_lock(&ntfa_cb.cb_lock);
>>> memset(&ntfa_cb.ntfs_mds_dest, 0, sizeof(MDS_DEST));
>>> - ntfa_cb.ntfs_up = 0;
>>> + ntfa_update_ntfsv_state(mds_cb_info->info.svc_evt.i_change ==
>>> NCSMDS_NO_ACTIVE ?
>>> + NTFA_NTFSV_NO_ACTIVE : NTFA_NTFSV_DOWN);
>>> pthread_mutex_unlock(&ntfa_cb.cb_lock);
>>> }
>>> break;
>>> @@ -382,7 +383,8 @@ static uint32_t ntfa_mds_svc_evt(struct
>>> TRACE_2("MSG from NTFS NCSMDS_NEW_ACTIVE/UP");
>>> pthread_mutex_lock(&ntfa_cb.cb_lock);
>>> ntfa_cb.ntfs_mds_dest = mds_cb_info->info.svc_evt.i_dest;
>>> - ntfa_cb.ntfs_up = 1;
>>> + ntfa_update_ntfsv_state(mds_cb_info->info.svc_evt.i_change ==
>>> NCSMDS_NEW_ACTIVE ?
>>> + NTFA_NTFSV_NEW_ACTIVE : NTFA_NTFSV_UP);
>>> if (ntfa_cb.ntfs_sync_awaited) {
>>> /* signal waiting thread */
>>> m_NCS_SEL_OBJ_IND(&ntfa_cb.ntfs_sync_sel);
>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>> b/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>> --- a/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>> @@ -49,7 +49,7 @@ static unsigned int ntfa_create(void)
>>> }
>>>
>>> /* Block and wait for indication from MDS meaning NTFS is up */
>>> - osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_sync_sel), 30000);
>>> + osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_sync_sel), 10000);
>> [Praveen] Did not get why it is changed to 10 seconds.
>> [Minh] I think 30 seconds is just too long
>>>
>>> pthread_mutex_lock(&ntfa_cb.cb_lock);
>>> ntfa_cb.ntfs_sync_awaited = 0;
>>> @@ -108,7 +108,25 @@ static bool ntfa_clear_mbx(NCSCONTEXT ar
>>> }
>>> return true;
>>> }
>>> +/****************************************************************************
>>>
>>> + Name : ntfa_notification_list_del
>>>
>>> + Description : This routine free element allocated in list of
>>> notification
>>> +
>>> + Arguments : pointer to the list of notification records anchor.
>>> +
>>> + Return Values : None
>>> +
>>> + Notes :
>>> +******************************************************************************/
>>>
>>> +static void ntfa_notification_list_del(ntfa_notification_hdl_rec_t
>>> **plstr_hdl)
>>> +{
>>> + ntfa_notification_hdl_rec_t *lstr_hdl = *plstr_hdl;
>>> + while (lstr_hdl != NULL) {
>>> + ntfa_notification_destructor(lstr_hdl);
>>> + lstr_hdl = lstr_hdl->next;
>>> + }
>>> +}
>>> /****************************************************************************
>>>
>>> Name : ntfa_notification_hdl_rec_list_del
>>>
>>> @@ -130,6 +148,70 @@ static void ntfa_notification_hdl_rec_li
>>> lstr_hdl = NULL;
>>> }
>>> }
>>> +/****************************************************************************
>>>
>>> + Name : ntfa_filter_list_del
>>> +
>>> + Description : This routine free element allocated in list of filter
>>> +
>>> + Arguments : pointer to the list of filters records anchor.
>>> +
>>> + Return Values : None
>>> +
>>> + Notes :
>>> +******************************************************************************/
>>>
>>> +static void ntfa_filter_list_del(ntfa_filter_hdl_rec_t **plstr_hdl)
>>> +{
>>> + ntfa_filter_hdl_rec_t *lstr_hdl = *plstr_hdl;
>>> +
>>> + while (lstr_hdl != NULL) {
>>> + ntfa_filter_destructor(lstr_hdl);
>>> + lstr_hdl = lstr_hdl->next;
>>> + }
>>> +}
>>> +/****************************************************************************
>>>
>>> + Name : ntfa_filter_hdl_rec_list_del
>>> +
>>> + Description : This routine deletes a list of allocated filters
>>> +
>>> + Arguments : pointer to the list of filters records anchor.
>>> +
>>> + Return Values : None
>>> +
>>> + Notes :
>>> +******************************************************************************/
>>>
>>> +static void ntfa_filter_hdl_rec_list_del(ntfa_filter_hdl_rec_t
>>> **plstr_hdl)
>>> +{
>>> + ntfa_filter_hdl_rec_t *lstr_hdl;
>>> +
>>> + while ((lstr_hdl = *plstr_hdl) != NULL) {
>>> + *plstr_hdl = lstr_hdl->next;
>>> + ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, lstr_hdl->filter_hdl);
>>> + free(lstr_hdl);
>>> + lstr_hdl = NULL;
>>> + }
>>> +}
>>> +/****************************************************************************
>>>
>>> + Name : ntfa_reader_hdl_rec_list_del
>>> +
>>> + Description : This routine deletes a list of allocated readers
>>> +
>>> + Arguments : pointer to the list of readers records anchor.
>>> +
>>> + Return Values : None
>>> +
>>> + Notes :
>>> +******************************************************************************/
>>>
>>> +static void ntfa_reader_hdl_rec_list_del(ntfa_reader_hdl_rec_t
>>> **plstr_hdl)
>>> +{
>>> + ntfa_reader_hdl_rec_t *lstr_hdl;
>>> + while ((lstr_hdl = *plstr_hdl) != NULL) {
>>> + *plstr_hdl = lstr_hdl->next;
>>> + ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, lstr_hdl->reader_hdl);
>>> + ntfa_del_ntf_filter_ptrs(&lstr_hdl->filters);
>>> + free(lstr_hdl);
>>> + lstr_hdl = NULL;
>>> + }
>>> +}
>>>
>>> static SaAisErrorT
>>> ntfa_alloc_callback_notification(SaNtfNotificationsT *notification,
>>> ntfsv_send_not_req_t *not_cbk,
>>> ntfa_client_hdl_rec_t *hdl_rec)
>>> @@ -407,6 +489,10 @@ static SaAisErrorT ntfa_hdl_cbk_rec_prc(
>>> free(cbk_info->param.discarded_cbk.discardedNotificationIdentifiers);
>>> }
>>> break;
>>> + case NTFSV_DUMMY_CALLBACK:
>>> + TRACE("Do nothing with dummy callback, just return OK");
>>> + rc = SA_AIS_OK;
>>> + break;
>>> default:
>>> TRACE("unsupported callback type: %d", cbk_info->type);
>>> rc = SA_AIS_ERR_LIBRARY;
>>> @@ -544,20 +630,21 @@ unsigned int ntfa_startup(void)
>>> *
>>> * @return unsigned int
>>> */
>>> -unsigned int ntfa_shutdown(void)
>>> +unsigned int ntfa_shutdown(bool forced)
>>> {
>>> unsigned int rc = NCSCC_RC_SUCCESS;
>>>
>>> - TRACE_ENTER2("ntfa_use_count: %u", ntfa_use_count);
>>> + TRACE_ENTER2("ntfa_use_count: %u, forced: %u", ntfa_use_count,
>>> forced);
>>> pthread_mutex_lock(&ntfa_lock);
>>>
>>> - if (ntfa_use_count > 1) {
>>> - /* Users still exist, just decrement the use count */
>>> - ntfa_use_count--;
>>> - } else if (ntfa_use_count == 1) {
>>> + if ((forced && (ntfa_use_count > 0)) || (ntfa_use_count == 1)) {
>>> ntfa_destroy();
>>> rc = ncs_agents_shutdown();
>>> ntfa_use_count = 0;
>>> + ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_NONE;
>>> + } else if (ntfa_use_count > 1) {
>>> + /* Users still exist, just decrement the use count */
>>> + ntfa_use_count--;
>>> }
>>>
>>> pthread_mutex_unlock(&ntfa_lock);
>>> @@ -633,7 +720,27 @@ ntfa_client_hdl_rec_t *ntfa_find_hdl_rec
>>>
>>> return NULL;
>>> }
>>> +/****************************************************************************
>>>
>>> + Name : ntfa_subscriber_list_del
>>>
>>> + Description : This routine deletes a list of allocated subscribers
>>> +
>>> + Arguments : pointer to the list of subscribers records anchor.
>>> +
>>> + Return Values : None
>>> +
>>> + Notes :
>>> +******************************************************************************/
>>>
>>> +void ntfa_subscriber_list_del()
>>> +{
>>> + ntfa_subscriber_list_t *listPtr = subscriberNoList;
>>> + while (listPtr != NULL) {
>>> + ntfa_subscriber_list_t* tmpSub = listPtr;
>>> + listPtr = listPtr->next;
>>> + free(tmpSub);
>>> + }
>>> + subscriberNoList = NULL;
>>> +}
>>> /****************************************************************************
>>>
>>> Name : ntfa_hdl_list_del
>>>
>>> @@ -651,15 +758,24 @@ void ntfa_hdl_list_del(ntfa_client_hdl_r
>>>
>>> while ((client_hdl = *p_client_hdl) != NULL) {
>>> *p_client_hdl = client_hdl->next;
>>> + m_NCS_IPC_DETACH(&client_hdl->mbx, ntfa_clear_mbx, NULL);
>>> + m_NCS_IPC_RELEASE(&client_hdl->mbx, NULL);
>>> ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, client_hdl->local_hdl);
>>> /** clean up the channel records for this ntfa-client
>>> **/
>>> + ntfa_notification_list_del(&client_hdl->notification_list);
>>> ntfa_notification_hdl_rec_list_del(&client_hdl->notification_list);
>>> +
>>> + ntfa_filter_list_del(&client_hdl->filter_list);
>>> + ntfa_filter_hdl_rec_list_del(&client_hdl->filter_list);
>>> +
>>> + ntfa_reader_hdl_rec_list_del(&client_hdl->reader_list);
>>> /** remove the association with hdl-mngr
>>> **/
>>> free(client_hdl);
>>> client_hdl = 0;
>>> }
>>> + ntfa_subscriber_list_del();
>>> }
>>>
>>> /****************************************************************************
>>>
>>> @@ -757,10 +873,80 @@ uint32_t ntfa_filter_hdl_rec_del(ntfa_fi
>>> TRACE("The node couldn't be deleted");
>>> return NCSCC_RC_FAILURE;
>>> }
>>> +/****************************************************************************
>>>
>>> + Name : ntfa_hdl_rec_force_del
>>>
>>> + Description : This routine deletes all memory allocated to client.
>>> +
>>> + Arguments : NTFA_CLIENT_HDL_REC **list_head
>>> + NTFA_CLIENT_HDL_REC *rm_node
>>> +
>>> + Return Values : None
>>> +
>>> + Notes :
>>> +******************************************************************************/
>>>
>>> +void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **list_head,
>>> ntfa_client_hdl_rec_t *rm_node)
>>> +{
>>> + ntfa_client_hdl_rec_t *list_iter = *list_head;
>>> + TRACE_ENTER();
>>> + /* First remove the rm_node out of the list of client */
>>> + if (list_iter == rm_node)
>>> + *list_head = rm_node->next;
>>> + else {
>>> + while (list_iter) {
>>> + if (list_iter->next == rm_node) {
>>> + list_iter->next = rm_node->next;
>>> + break;
>>> + }
>>> + list_iter = list_iter->next;
>>> + }
>>> + }
>>> + /* Release all msgs in mailbox */
>>> + ntfsv_msg_t *cbk_msg;
>>> + while((cbk_msg =
>>> (ntfsv_msg_t*)m_NCS_IPC_NON_BLK_RECEIVE(&rm_node->mbx, cbk_msg))
>>> + != NULL) {
>>> + ntfa_msg_destroy(cbk_msg);
>>> + }
>>> + /* delete subscriber of this client out of the subcriberNoList*/
>>> + ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
>>> + while (subscriber_hdl != NULL) {
>>> + ntfa_subscriber_list_t *rm_subscriber = subscriber_hdl;
>>> + subscriber_hdl = subscriber_hdl->next;
>>> + if (rm_node->local_hdl ==
>>> rm_subscriber->subscriberListNtfHandle) {
>>> + if (rm_subscriber->next != NULL) {
>>> + rm_subscriber->next->prev = rm_subscriber->prev;
>>> + }
>>> +
>>> + if (rm_subscriber->prev != NULL) {
>>> + rm_subscriber->prev->next = rm_subscriber->next;
>>> + } else {
>>> + if (rm_subscriber->next != NULL)
>>> + subscriberNoList = rm_subscriber->next;
>>> + else
>>> + subscriberNoList = NULL;
>>> + }
>>> + ntfa_del_ntf_filter_ptrs(&rm_subscriber->filters);
>>> + free(rm_subscriber);
>>> + }
>>> + }
>>> + /* Now delete client */
>>> + m_NCS_IPC_DETACH(&rm_node->mbx, ntfa_clear_mbx, NULL);
>>> + m_NCS_IPC_RELEASE(&rm_node->mbx, NULL);
>>> + ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node->local_hdl);
>>> + ntfa_notification_list_del(&rm_node->notification_list);
>>> + ntfa_notification_hdl_rec_list_del(&rm_node->notification_list);
>>> +
>>> + ntfa_filter_list_del(&rm_node->filter_list);
>>> + ntfa_filter_hdl_rec_list_del(&rm_node->filter_list);
>>> +
>>> + ntfa_reader_hdl_rec_list_del(&rm_node->reader_list);
>>> + free(rm_node);
>>> +
>>> + TRACE_LEAVE();
>>> +}
>>> /****************************************************************************
>>>
>>> Name : ntfa_hdl_rec_del
>>> -
>>> +
>>> Description : This routine deletes the a client handle record from
>>> a list of client hdl records.
>>>
>>> @@ -962,7 +1148,7 @@ ntfa_client_hdl_rec_t *ntfa_hdl_rec_add(
>>> /** Associate with the client_id obtained from NTFS
>>> **/
>>> rec->ntfs_client_id = client_id;
>>> -
>>> + rec->valid = true;
>>> /** Initialize and attach the IPC/Priority queue
>>> **/
>>>
>>> @@ -1063,7 +1249,7 @@ static void logtrace_init_constructor(vo
>>> *
>>> * @param instance
>>> */
>>> -void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t *instance)
>>> +void ntfa_notification_destructor(ntfa_notification_hdl_rec_t
>>> *instance)
>>> {
>>> ntfa_notification_hdl_rec_t *notificationInstance = instance;
>>>
>>> @@ -1111,7 +1297,7 @@ void ntfa_hdl_rec_destructor(ntfa_notifi
>>> *
>>> * @param instance
>>> */
>>> -void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t *filter_rec)
>>> +void ntfa_filter_destructor(ntfa_filter_hdl_rec_t *filter_rec)
>>> {
>>> switch (filter_rec->ntfType) {
>>> case SA_NTF_TYPE_OBJECT_CREATE_DELETE:
>>> @@ -1202,6 +1388,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>>> **/
>>> ncshm_give_hdl(rm_node->reader_hdl);
>>> ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node->reader_hdl);
>>> + ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>>> free(rm_node);
>>> return NCSCC_RC_SUCCESS;
>>> } else { /* find the rec */
>>> @@ -1213,6 +1400,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>>> **/
>>> ncshm_give_hdl(rm_node->reader_hdl);
>>> ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA,
>>> rm_node->reader_hdl);
>>> + ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>>> free(rm_node);
>>> return NCSCC_RC_SUCCESS;
>>> }
>>> @@ -1258,3 +1446,258 @@ void ntfa_add_to_async_cbk_msg_list(ntfs
>>>
>>> TRACE_LEAVE();
>>> }
>>> +
>>> +/****************************************************************************
>>>
>>> + Name : ntfa_notify_handle_invalid
>>> +
>>> + Description : This routine sends a dummy callback msg to
>>> client's mailbox
>>> + so that the client polls in and calls saNtfDispatch.
>>> + Arguments :
>>> +
>>> + Return Values : None
>>> +
>>> + Notes : None
>>> +******************************************************************************/
>>>
>>> +void ntfa_notify_handle_invalid() {
>>> + ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
>>> + TRACE_ENTER();
>>> + while (client_hdl != NULL) {
>>> + /* Only applicable for subscriber */
>>> + if (client_hdl->reg_cbk.saNtfNotificationCallback != NULL ||
>>> + client_hdl->reg_cbk.saNtfNotificationDiscardedCallback != NULL) {
>>> + /* Create a dummy msg */
>>> + ntfsv_msg_t *msg = malloc(sizeof(ntfsv_msg_t));
>>> + memset(msg, 0, sizeof(ntfsv_msg_t));
>>> + msg->info.cbk_info.type = NTFSV_DUMMY_CALLBACK;
>>> + /* Send dummy msg to client mailbox */
>> [Praveen] Is this dummy callback being sent to client in a way to
>> force it to recover client and subscriber?
> [Minh] Because agent doesn't recover client in mds callback, so in case
> of subscriber who is just waiting for notification, agent needs to send
> dummy callback to client's mailbox to wake up client's polling loop.
> Then client will call saAmfDispatch, and from there agent will recover
> client handles. This dummy callback is invisible to client.
>>> + if (m_NCS_IPC_SEND(&client_hdl->mbx, msg,
>>> MDS_SEND_PRIORITY_HIGH) != NCSCC_RC_SUCCESS) {
>>> + TRACE_1("m_NCS_IPC_SEND Failed to client(id:%u)",
>>> client_hdl->ntfs_client_id);
>>> + ntfa_msg_destroy(msg);
>>> + }
>>> + }
>>> +
>>> + client_hdl = client_hdl->next;
>>> + }
>>> + TRACE_LEAVE();
>>> +}
>>> +/****************************************************************************
>>>
>>> + Name : ntfa_update_ntfsv_state
>>> +
>>> + Description : Update current NTF Server state by the
>>> @changedState indicated
>>> + by MDS event
>>> + Arguments : @changedState [IN]: state to be changed of NTF Server
>>> +
>>> + Return Values : None
>>> +
>>> + Notes : None
>>> +******************************************************************************/
>>>
>>> +void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState)
>>> +{
>>> + TRACE_ENTER();
>>> + TRACE_1("Current state: %u, Changed state: %u",
>>> ntfa_cb.ntfa_ntfsv_state,
>>> + changedState);
>>> +
>>> + ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
>>> +
>>> + switch (ntfa_cb.ntfa_ntfsv_state){
>>> + case NTFA_NTFSV_NONE:
>>> + ntfa_cb.ntfa_ntfsv_state = changedState;
>>> + break;
>>> + case NTFA_NTFSV_DOWN:
>>> + if (changedState == NTFA_NTFSV_NEW_ACTIVE ||
>>> + changedState == NTFA_NTFSV_UP) {
>>> + TRACE("Active NTF server has been restarted");
>>> + ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_UP;
>>> + ntfa_notify_handle_invalid();
>>> + } else
>>> + TRACE("Unexpected state changes");
>>> + break;
>>> + case NTFA_NTFSV_NO_ACTIVE:
>>> + if (changedState == NTFA_NTFSV_NEW_ACTIVE) {
>>> + TRACE("Standby NTF server becomes new Active");
>>> + /* NTF server is functioning normally */
>>> + ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_UP;
>>> + } else if (changedState == NTFA_NTFSV_DOWN) {
>>> + TRACE("Active NTF Server is Down");
>>> + ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_DOWN;
>>> + /* Mark all client handles are invalid */
>>> + while (client_hdl != NULL) {
>>> + client_hdl->valid = false;
>>> + client_hdl = client_hdl->next;
>>> + }
>>> + }
>>> + break;
>>> + case NTFA_NTFSV_NEW_ACTIVE:
>>> + TRACE("Unknown");
>>> + break;
>>> + case NTFA_NTFSV_UP:
>>> + if (changedState == NTFA_NTFSV_NO_ACTIVE) {
>>> + TRACE("Active NTF server temporarily unavailable");
>>> + /* Failover/Switchover is happening
>>> + * Any API calls result in TRY_AGAIN
>>> + */
>>> + ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_NO_ACTIVE;
>>> + } else
>>> + TRACE("Unexpected state changes");
>>> + break;
>>> + default:
>>> + osafassert(false);
>>> + }
>>> +
>>> + TRACE_LEAVE();
>>> +}
>>> +/****************************************************************************
>>>
>>> + Name : ntfa_copy_ntf_filter_ptrs
>>> +
>>> + Description : Copy a list of filter from pSrc to pDes
>>> +
>>> + Arguments : pDes* [OUT]: list of outcome filter
>>> + pSrc* [IN]: list of input filter
>>> +
>>> + Return Values : SA_AIS_OK if succeed, other values as failed
>>> +
>>> + Notes : None
>>> +******************************************************************************/
>>>
>>> +SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
>>> + const ntfsv_filter_ptrs_t* pSrc) {
>>> + SaAisErrorT rc = SA_AIS_OK;
>>> + SaNtfNotificationFilterHeaderT *des_header;
>>> + SaNtfNotificationFilterHeaderT *src_header;
>>> + TRACE_ENTER();
>>> +
>>> + if (pSrc->alarm_filter) {
>>> + pDes->alarm_filter = calloc(1,
>>> sizeof(SaNtfAlarmNotificationFilterT));
>>> + des_header = &(pDes->alarm_filter->notificationFilterHeader);
>>> + src_header = &(pSrc->alarm_filter->notificationFilterHeader);
>>> + if ((rc = ntfsv_filter_header_alloc(des_header,
>>> src_header->numEventTypes,
>>> + src_header->numNotificationObjects,
>>> + src_header->numNotifyingObjects,
>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>> + goto done;
>>> + if ((rc = ntfsv_filter_alarm_alloc(pDes->alarm_filter,
>>> + pSrc->alarm_filter->numProbableCauses,
>>> + pSrc->alarm_filter->numPerceivedSeverities,
>>> + pSrc->alarm_filter->numTrends)) != SA_AIS_OK)
>>> + goto done;
>>> + if ((rc = ntfsv_copy_ntf_filter_alarm(pDes->alarm_filter,
>>> + pSrc->alarm_filter)) != SA_AIS_OK)
>>> + goto done;
>>> + }
>>> +
>>> + if (pSrc->sec_al_filter) {
>>> + pDes->sec_al_filter = calloc(1,
>>> sizeof(SaNtfSecurityAlarmNotificationFilterT));
>>> + des_header = &(pDes->sec_al_filter->notificationFilterHeader);
>>> + src_header = &(pSrc->sec_al_filter->notificationFilterHeader);
>>> + if ((rc = ntfsv_filter_header_alloc(des_header,
>>> src_header->numEventTypes,
>>> + src_header->numNotificationObjects,
>>> + src_header->numNotifyingObjects,
>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>> + goto done;
>>> + if ((rc = ntfsv_filter_sec_alarm_alloc(pDes->sec_al_filter,
>>> + pSrc->sec_al_filter->numProbableCauses,
>>> + pSrc->sec_al_filter->numSeverities,
>>> + pSrc->sec_al_filter->numSecurityAlarmDetectors,
>>> + pSrc->sec_al_filter->numServiceUsers,
>>> + pSrc->sec_al_filter->numServiceProviders)) != SA_AIS_OK)
>>> + goto done;
>>> + if ((rc = ntfsv_copy_ntf_filter_sec_alarm(pDes->sec_al_filter,
>>> + pSrc->sec_al_filter)) != SA_AIS_OK)
>>> + goto done;
>>> + }
>>> +
>>> + if (pSrc->sta_ch_filter) {
>>> + pDes->sta_ch_filter = calloc(1,
>>> sizeof(SaNtfStateChangeNotificationFilterT));
>>> + des_header = &(pDes->sta_ch_filter->notificationFilterHeader);
>>> + src_header = &(pSrc->sta_ch_filter->notificationFilterHeader);
>>> + if ((rc = ntfsv_filter_header_alloc(des_header,
>>> src_header->numEventTypes,
>>> + src_header->numNotificationObjects,
>>> + src_header->numNotifyingObjects,
>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>> + goto done;
>>> + if ((rc = ntfsv_filter_state_ch_alloc(pDes->sta_ch_filter,
>>> + pSrc->sta_ch_filter->numSourceIndicators,
>>> + pSrc->sta_ch_filter->numStateChanges)) != SA_AIS_OK)
>>> + goto done;
>>> + if ((rc = ntfsv_copy_ntf_filter_state_ch(pDes->sta_ch_filter,
>>> + pSrc->sta_ch_filter)) != SA_AIS_OK)
>>> + goto done;
>>> + }
>>> +
>>> + if (pSrc->obj_cr_del_filter) {
>>> + pDes->obj_cr_del_filter = calloc(1,
>>> sizeof(SaNtfObjectCreateDeleteNotificationFilterT));
>>> + des_header =
>>> &(pDes->obj_cr_del_filter->notificationFilterHeader);
>>> + src_header =
>>> &(pSrc->obj_cr_del_filter->notificationFilterHeader);
>>> + if ((rc = ntfsv_filter_header_alloc(des_header,
>>> src_header->numEventTypes,
>>> + src_header->numNotificationObjects,
>>> + src_header->numNotifyingObjects,
>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>> + goto done;
>>> + if ((rc =
>>> ntfsv_filter_obj_cr_del_alloc(pDes->obj_cr_del_filter,
>>> + pSrc->obj_cr_del_filter->numSourceIndicators)) != SA_AIS_OK)
>>> + goto done;
>>> + if ((rc =
>>> ntfsv_copy_ntf_filter_obj_cr_del(pDes->obj_cr_del_filter,
>>> + pSrc->obj_cr_del_filter)) != SA_AIS_OK)
>>> + goto done;
>>> + }
>>> +
>>> + if (pSrc->att_ch_filter) {
>>> + pDes->att_ch_filter = calloc(1,
>>> sizeof(SaNtfAttributeChangeNotificationFilterT));
>>> + des_header = &(pDes->att_ch_filter->notificationFilterHeader);
>>> + src_header = &(pSrc->att_ch_filter->notificationFilterHeader);
>>> + if ((rc = ntfsv_filter_header_alloc(des_header,
>>> src_header->numEventTypes,
>>> + src_header->numNotificationObjects,
>>> + src_header->numNotifyingObjects,
>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>> + goto done;
>>> + if ((rc = ntfsv_filter_attr_change_alloc(pDes->att_ch_filter,
>>> + pSrc->att_ch_filter->numSourceIndicators)) != SA_AIS_OK)
>>> + goto done;
>>> + if ((rc = ntfsv_copy_ntf_filter_attr_ch(pDes->att_ch_filter,
>>> + pSrc->att_ch_filter)) != SA_AIS_OK)
>>> + goto done;
>>> + }
>>> +done:
>>> + TRACE_LEAVE();
>>> + return rc;
>>> +}
>>> +
>>> +/****************************************************************************
>>>
>>> + Name : ntfa_del_ntf_filter_ptrs
>>> +
>>> + Description : Delete the filter pointers
>>> +
>>> + Arguments : filter_ptrs* [IN/OUT]: filter pointers
>>> +
>>> + Return Values : SA_AIS_OK if succeed, other values as failed
>>> +
>>> + Notes : None
>>> +******************************************************************************/
>>>
>>> +SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t* filter_ptrs)
>>> +{
>>> + SaAisErrorT rc = SA_AIS_OK;
>>> + if (filter_ptrs->alarm_filter) {
>>> + ntfsv_filter_alarm_free(filter_ptrs->alarm_filter, true);
>>> + free(filter_ptrs->alarm_filter);
>>> + }
>>> +
>>> + if (filter_ptrs->sec_al_filter) {
>>> + ntfsv_filter_sec_alarm_free(filter_ptrs->sec_al_filter, true);
>>> + free(filter_ptrs->sec_al_filter);
>>> + }
>>> +
>>> + if (filter_ptrs->sta_ch_filter) {
>>> + ntfsv_filter_state_ch_free(filter_ptrs->sta_ch_filter, true);
>>> + free(filter_ptrs->sta_ch_filter);
>>> + }
>>> +
>>> + if (filter_ptrs->obj_cr_del_filter) {
>>> + ntfsv_filter_obj_cr_del_free(filter_ptrs->obj_cr_del_filter, true);
>>> + free(filter_ptrs->obj_cr_del_filter);
>>> + }
>>> +
>>> + if (filter_ptrs->att_ch_filter) {
>>> + ntfsv_filter_attr_ch_free(filter_ptrs->att_ch_filter, true);
>>> + free(filter_ptrs->att_ch_filter);
>>> + }
>>> + return rc;
>>> +}
>>>
>>
>
------------------------------------------------------------------------------
Site24x7 APM Insight: Get Deep Visibility into Application Performance
APM + Mobile APM + RUM: Monitor 3 App instances at just $35/Month
Monitor end-to-end web transactions and take corrective actions now
Troubleshoot faster and improve end-user experience. Signup Now!
http://pubads.g.doubleclick.net/gampad/clk?id=272487151&iu=/4140
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel
