Hi Minh,
Some more comments:
-Recovery of client is not being done in thread safe way. For a single
client, multiple client_ids may get generated in a multi-threaded
application.
-In patch 4,
a)ntftest works with -ve option but in tag mode. As per help tag mode
is invoked when -t is given.
b)Also ntftest dows not work with -vpe and prints only -help output.
I do not have any further comments other.
Thanks,
Praveen
On 18-Feb-16 6:31 PM, praveen malviya wrote:
> Please see the responses with [Praveen]
>
> Thanks,
> Praveen
>
> On 12-Feb-16 8:52 PM, minh chau wrote:
>> Hi Praveen,
>>
>> Please find my comments inline [Minh]
>>
>> Thanks,
>> Minh
>>
>> On 12/02/16 22:10, praveen malviya wrote:
>>> Hi Minh,
>>>
>>> Please find some initial comments and questions marked with [Praveen].
>>>
>>> One question regarding the approach.
>>> In most of the changed APIs from saNtfInitialize() to
>>> saNtfNotificationReadFinalize(), clients are being recovered. Why the
>>> idea of recovering them is not considered in mds callback itself in
>>> the function call ntfa_update_ntfsv_state() when server state changes
>>> from DOWN to UP. It will save changes in APIs for recovering the
>>> clients. Clients are being marked invalid in the mds_callback itself.
>>>
>> [Minh] From memory I did try to recover clients in mds callback
>> (svc_evt) where indicates server UP. Though recovery needs to send
>> messages (reintroducing client id, read id, ...) to server, that will
>> end up another mds enc callback inside of first callback. That results
>> into fail to send *recovery* messages, there are two mds callback but
>> only one return. To solve this, I need to start a thread the recover
>> clients in background. That would cause another problem where up calls
>> from client could be blocked, since the recovery thread needs to block
>> mutual resource handles.
>> Another realistic reason, client can give up to send/read/receive
>> notification after receiving couple times of TRY_AGAIN due to server
>> down, and doing nothing until finalize handle. And after client stops
>> TRY_AGAIN loop, server can be restarted. In such cases, no needs to
>> recover client handles.
>>> Thanks,
>>> Praveen
>>>
>>> On 23-Dec-15 9:32 AM, Minh Hon Chau wrote:
>>>> osaf/libs/agents/saf/ntfa/ntfa.h | 31 +-
>>>> osaf/libs/agents/saf/ntfa/ntfa_api.c | 672
>>>> +++++++++++++++++++++++++++------
>>>> osaf/libs/agents/saf/ntfa/ntfa_mds.c | 6 +-
>>>> osaf/libs/agents/saf/ntfa/ntfa_util.c | 465 ++++++++++++++++++++++-
>>>> 4 files changed, 1022 insertions(+), 152 deletions(-)
>>>>
>>>>
>>>> The patch contains support for cloud resilience feature
>>>> in NTF Agent code. Please refer README.HYDRA for content
>>>> of the changes
>>>>
>>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa.h
>>>> b/osaf/libs/agents/saf/ntfa/ntfa.h
>>>> --- a/osaf/libs/agents/saf/ntfa/ntfa.h
>>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa.h
>>>> @@ -91,6 +91,7 @@ typedef struct ntfa_filter_hdl_rec {
>>>> typedef struct subscriberList {
>>>> SaNtfHandleT subscriberListNtfHandle;
>>>> SaNtfSubscriptionIdT subscriberListSubscriptionId;
>>>> + ntfsv_filter_ptrs_t filters; /* remember the filters used by
>>>> this subscriber */
>>>> struct subscriberList *prev;
>>>> struct subscriberList *next;
>>>> } ntfa_subscriber_list_t;
>>>> @@ -100,6 +101,10 @@ typedef struct ntfa_reader_hdl_rec {
>>>> unsigned int reader_id; /* handle value returned by NTFS for
>>>> this client */
>>>> SaNtfHandleT ntfHandle;
>>>> unsigned int reader_hdl; /* READER handle from handle mgr */
>>>> +
>>>> + ntfsv_filter_ptrs_t filters; /* remember the filters used by
>>>> this reader */
>>>> + SaNtfSearchCriteriaT searchCriteria; /* remember the
>>>> searchCriteria for recovery */
>>>> +
>>>> struct ntfa_reader_hdl_rec *next; /* next pointer for the
>>>> list in ntfa_cb_t */
>>>> struct ntfa_client_hdl_rec *parent_hdl; /* Back Pointer to
>>>> the client instantiation */
>>>> } ntfa_reader_hdl_rec_t;
>>>> @@ -114,24 +119,35 @@ typedef struct ntfa_client_hdl_rec {
>>>> ntfa_reader_hdl_rec_t *reader_list;
>>>> SYSF_MBX mbx; /* priority q mbx b/w MDS & Library */
>>>> struct ntfa_client_hdl_rec *next; /* next pointer for the
>>>> list in ntfa_cb_t */
>>>> + bool valid; /* handle is valid if it's known by NTF
>>>> server, used for headless hydra */
>>>> + SaVersionT version; /* the API version is being used by client,
>>>> used for recover after headless */
>>>> } ntfa_client_hdl_rec_t;
>>>>
>>>> /*
>>>> * The NTFA control block is the master anchor structure for all NTFA
>>>> * instantiations within a process.
>>>> */
>>>> +typedef enum {
>>>> + NTFA_NTFSV_NONE = 0,
>>>> + NTFA_NTFSV_DOWN,
>>>> + NTFA_NTFSV_NO_ACTIVE,
>>>> + NTFA_NTFSV_NEW_ACTIVE,
>>>> + NTFA_NTFSV_UP
>>>> +}ntfa_ntfsv_state_t;
>>>> +
>>>> typedef struct {
>>>> pthread_mutex_t cb_lock; /* CB lock */
>>>> ntfa_client_hdl_rec_t *client_list; /* NTFA client handle
>>>> database */
>>>> ntfa_reader_hdl_rec_t *reader_list;
>>>> MDS_HDL mds_hdl; /* MDS handle */
>>>> MDS_DEST ntfs_mds_dest; /* NTFS absolute/virtual address */
>>>> - int ntfs_up; /* Indicate that MDS subscription
>>>> - * is complete */
>>>> +
>>>> /* NTFS NTFA sync params */
>>>> int ntfs_sync_awaited;
>>>> NCS_SEL_OBJ ntfs_sync_sel;
>>>> SaUint32T ntf_var_data_limit; /* max allowed
>>>> variableDataSize */
>>>> + /* NTF Server state */
>>>> + ntfa_ntfsv_state_t ntfa_ntfsv_state;
>>>> } ntfa_cb_t;
>>>>
>>>> /* ntfa_saf_api.c */
>>>> @@ -149,7 +165,7 @@ extern void ntfsv_ntfa_evt_free(struct n
>>>>
>>>> /* ntfa_init.c */
>>>> extern unsigned int ntfa_startup(void);
>>>> -extern unsigned int ntfa_shutdown(void);
>>>> +extern unsigned int ntfa_shutdown(bool forced);
>>>>
>>>> /* ntfa_hdl.c */
>>>> extern SaAisErrorT ntfa_hdl_cbk_dispatch(ntfa_cb_t *,
>>>> ntfa_client_hdl_rec_t *, SaDispatchFlagsT);
>>>> @@ -159,6 +175,7 @@ extern ntfa_notification_hdl_rec_t *ntfa
>>>> extern ntfa_filter_hdl_rec_t
>>>> *ntfa_filter_hdl_rec_add(ntfa_client_hdl_rec_t **hdl_rec);
>>>> extern void ntfa_hdl_list_del(ntfa_client_hdl_rec_t **);
>>>> extern uint32_t ntfa_hdl_rec_del(ntfa_client_hdl_rec_t **,
>>>> ntfa_client_hdl_rec_t *);
>>>> +extern void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **,
>>>> ntfa_client_hdl_rec_t *);
>>>> extern uint32_t
>>>> ntfa_notification_hdl_rec_del(ntfa_notification_hdl_rec_t **,
>>>> ntfa_notification_hdl_rec_t *);
>>>> extern uint32_t ntfa_filter_hdl_rec_del(ntfa_filter_hdl_rec_t **,
>>>> ntfa_filter_hdl_rec_t *);
>>>> extern bool ntfa_validate_ntfa_client_hdl(ntfa_cb_t *ntfa_cb,
>>>> ntfa_client_hdl_rec_t *find_hdl_rec);
>>>> @@ -166,11 +183,15 @@ extern bool ntfa_validate_ntfa_client_hd
>>>> /* ntfa_util.c */
>>>> extern ntfa_client_hdl_rec_t
>>>> *ntfa_find_hdl_rec_by_client_id(ntfa_cb_t *ntfa_cb, uint32_t client_id);
>>>> extern void ntfa_msg_destroy(ntfsv_msg_t *msg);
>>>> -extern void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t
>>>> *instance);
>>>> -extern void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t
>>>> +extern void ntfa_notification_destructor(ntfa_notification_hdl_rec_t
>>>> *instance);
>>>> +extern void ntfa_filter_destructor(ntfa_filter_hdl_rec_t
>>>> *notificationFilterInstance);
>>>> extern ntfa_reader_hdl_rec_t
>>>> *ntfa_reader_hdl_rec_add(ntfa_client_hdl_rec_t **hdl_rec);
>>>> extern uint32_t ntfa_reader_hdl_rec_del(ntfa_reader_hdl_rec_t **,
>>>> ntfa_reader_hdl_rec_t *);
>>>> extern void ntfa_add_to_async_cbk_msg_list(ntfsv_msg_t ** head,
>>>> ntfsv_msg_t * new_node);
>>>> extern uint32_t ntfa_ntfs_msg_proc(ntfa_cb_t *cb, ntfsv_msg_t
>>>> *ntfsv_msg, MDS_SEND_PRIORITY_TYPE prio);
>>>> +extern void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState);
>>>> +extern SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
>>>> + const ntfsv_filter_ptrs_t* pSrc);
>>>> +extern SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t*
>>>> filter_ptrs);
>>>> #endif /* !NTFA_H */
>>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>>> b/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>>> --- a/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_api.c
>>>> @@ -34,11 +34,42 @@
>>>> /* The main controle block */
>>>> ntfa_cb_t ntfa_cb = {
>>>> .cb_lock = PTHREAD_MUTEX_INITIALIZER,
>>>> + .ntfa_ntfsv_state = NTFA_NTFSV_NONE,
>>>> };
>>>>
>>>> /* list of subscriptions for this process */
>>>> ntfa_subscriber_list_t *subscriberNoList = NULL;
>>>>
>>>> +/*
>>>> + * @Brief: Determine the common "availability" of API, which depends
>>>> on the
>>>> + * NTF Server state
>>>> + * @Param: None
>>>> + * @Return: OK - If Server state is UP
>>>> + * TRY_AGAIN - If Server state is No Active (temporary no
>>>> active)
>>>> + * BAD_HANDLE - If Server state is completely Down (no
>>>> active at all).
>>>> + * This error code will be returned to client
>>>> calling API
>>>> + */
>>>> +static SaAisErrorT checkNtfServerState()
>>>> +{
>>>> + SaAisErrorT rc = SA_AIS_ERR_NOT_SUPPORTED;
>>>> + /* Check NTF server availability */
>>>> + if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_UP) {
>>>> + rc = SA_AIS_OK;
>>>> + } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_DOWN) {
>>>> + TRACE("NTFS server is down");
>>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>> [Praveen] 1. Function header says BAD_HANDLE for "down" but here
>>> TRY_AGAIN is returned.
>> [Minh]: I got similar comments from Lennart, I will correct the function
>> description
>>> 2. If server state is down, it means both ntfs are not up which means
>>> headless cluster. But in case of normal cluster also application will
>>> get TRY_AGAIN. Are you relying on the fact that in normal
>>> cluster case payloads will be rebooted? if answer is yes then what if
>>> reboot is delayed by changing the opensaf_reboot script which is
>>> customizable?
>> [Minh]: I don't get your question. No impact if payloads reboot
> [Praveen] Actually I want draw attention to two problems here:
> 1) Normal cluster case: Both controllers are up in normal cluster
> (attribute commented in immd.conf) and some ntf application is running
> on the payload. Now if opensaf is stopped on the payload, application
> will still get TRY_AGIAN. Now if a fresh controller is started and
> payload is not started, the application API will be successful. But this
> application should not be able to get a success for any API since its
> payload is down.
> 2) Headless cluster case: Same case when with headless cluster
> configured (attribute un-commented in immd.conf).
>
> I think this may be the generic problem with all the service with two
> tier architecture and a fix is needed. There should be a mechanism
> though which agent should detect opensaf shutdown on local node and
> clear all clients data.
>
>
>>>
>>>> + } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NO_ACTIVE) {
>>>> + TRACE("NTFS server is unavailable");
>>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>>> + } else if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NONE) {
>>>> + TRACE("No NTF server is detected, or API is called during
>>>> headless");
>>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>>> + } else
>>>> + TRACE("Not supported API call under NTF Server state (%u)",
>>>> + ntfa_cb.ntfa_ntfsv_state);
>>>> + return rc;
>>>> +}
>>>> +
>>>> static SaAisErrorT checkNtfValueTypeRange(SaNtfValueTypeT type)
>>>> {
>>>> return (type < SA_NTF_VALUE_UINT8 || type >
>>>> SA_NTF_VALUE_ARRAY)? SA_AIS_ERR_INVALID_PARAM
>>>> @@ -871,7 +902,266 @@ SaAisErrorT getFilters(const SaNtfNotifi
>>>> }
>>>>
>>>> /* end help functions */
>>>> -
>>>> +/****************************************************************************
>>>>
>>>> + Name : reinitializeClient
>>>> +
>>>> + Description : This routine sends initialize messages to NTF
>>>> Server in
>>>> + order to obtain new client id
>>>> +
>>>> + Arguments : client_hdl* [IN]: client handle
>>>> +
>>>> + Return Values : SA_AIS_OK if success, others are failure
>>>> +
>>>> + Notes : None
>>>> +******************************************************************************/
>>>>
>>>> +SaAisErrorT reinitializeClient(ntfa_client_hdl_rec_t* client_hdl) {
>>>> + uint32_t mds_rc;
>>>> + SaAisErrorT rc = SA_AIS_OK;
>>>> + ntfsv_msg_t i_msg, *o_msg = NULL;
>>>> +
>>>> + TRACE_ENTER();
>>>> +
>>>> + memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>>>> + i_msg.type = NTFSV_NTFA_API_MSG;
>>>> + i_msg.info.api_info.type = NTFSV_INITIALIZE_REQ;
>>>> + i_msg.info.api_info.param.init.version = client_hdl->version;
>>>> +
>>>> + mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>>>> NTFS_WAIT_TIME);
>>>> + switch (mds_rc) {
>>>> + case NCSCC_RC_SUCCESS:
>>>> + break;
>>>> + case NCSCC_RC_REQ_TIMOUT:
>>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>>> + goto done;
>>>> + default:
>>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>>> + goto done;
>>>> + }
>>>> +
>>>> + osafassert(o_msg != NULL);
>>>> + /* Check response msg: type, rc */
>>>> + if (o_msg->info.api_resp_info.type != NTFSV_INITIALIZE_RSP) {
>>>> + TRACE("info.api_resp_info.type:%u",
>>>> o_msg->info.api_resp_info.type);
>>>> + rc = SA_AIS_ERR_LIBRARY;
>>>> + goto done;
>>>> + }
>>>> + if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>>>> + TRACE("info.api_resp_info.rc:%u",
>>>> o_msg->info.api_resp_info.rc);
>>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>>> + goto done;
>>>> + }
>>>> +
>>>> + client_hdl->ntfs_client_id =
>>>> o_msg->info.api_resp_info.param.init_rsp.client_id;
>>>> + TRACE("Successfully recover client_id, new client_id:%u",
>>>> + client_hdl->ntfs_client_id);
>>>> +
>>>> +done:
>>>> + if (o_msg)
>>>> + ntfa_msg_destroy(o_msg);
>>>> + TRACE_LEAVE();
>>>> + return rc;
>>>> +}
>>>> +/****************************************************************************
>>>>
>>>> + Name : recoverReader
>>>> +
>>>> + Description : This routine sends read_initialize messages to NTF
>>>> Server in
>>>> + order to reintroduce this reader to NTF Server
>>>> +
>>>> + Arguments : client_hdl* [IN]: client handle
>>>> + reader_hdl* [IN]: reader handle
>>>> +
>>>> + Return Values : SA_AIS_OK if success, others are failure
>>>> +
>>>> + Notes : None
>>>> +******************************************************************************/
>>>>
>>>> +SaAisErrorT recoverReader(ntfa_client_hdl_rec_t* client_hdl,
>>>> ntfa_reader_hdl_rec_t* reader_hdl) {
>>>> + uint32_t mds_rc;
>>>> + SaAisErrorT rc = SA_AIS_OK;
>>>> + ntfsv_msg_t i_msg, *o_msg = NULL;
>>>> + ntfsv_reader_init_req_2_t *send_param;
>>>> +
>>>> + TRACE_ENTER();
>>>> +
>>>> + /* Make sure the filters have not been deleted */
>>>> + if (reader_hdl->filters.alarm_filter == NULL &&
>>>> + reader_hdl->filters.att_ch_filter == NULL &&
>>>> + reader_hdl->filters.obj_cr_del_filter == NULL &&
>>>> + reader_hdl->filters.sec_al_filter == NULL &&
>>>> + reader_hdl->filters.sta_ch_filter == NULL)
>>>> + return SA_AIS_ERR_BAD_HANDLE;
>>>> +
>>>> + memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>>>> + i_msg.type = NTFSV_NTFA_API_MSG;
>>>> + i_msg.info.api_info.type = NTFSV_READER_INITIALIZE_REQ_2;
>>>> + send_param = &i_msg.info.api_info.param.reader_init_2;
>>>> + send_param->head.client_id = client_hdl->ntfs_client_id;
>>>> + send_param->head.searchCriteria = reader_hdl->searchCriteria;
>>>> + send_param->f_rec = reader_hdl->filters;
>>>> +
>>>> + mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>>>> NTFS_WAIT_TIME);
>>>> +
>>>> + switch (mds_rc) {
>>>> + case NCSCC_RC_SUCCESS:
>>>> + break;
>>>> + case NCSCC_RC_REQ_TIMOUT:
>>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>>> + goto done;
>>>> + default:
>>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>>> + goto done;
>>>> + }
>>>> +
>>>> + osafassert(o_msg != NULL);
>>>> + if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>>>> + TRACE("o_msg->info.api_resp_info.rc:%u",
>>>> o_msg->info.api_resp_info.rc);
>>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>>> + goto done;
>>>> + }
>>>> +
>>>> + if (o_msg->info.api_resp_info.type !=
>>>> NTFSV_READER_INITIALIZE_RSP) {
>>>> + TRACE("msg type (%d) failed",
>>>> (int)o_msg->info.api_resp_info.type);
>>>> + rc = SA_AIS_ERR_LIBRARY;
>>>> + goto done;
>>>> + }
>>>> + /* Update reader_id since it may be changed */
>>>> + reader_hdl->reader_id =
>>>> o_msg->info.api_resp_info.param.reader_init_rsp.readerId;
>>>> +
>>>> + TRACE("Recover reader successfully");
>>>> +done:
>>>> + if (o_msg)
>>>> + ntfa_msg_destroy(o_msg);
>>>> + TRACE_LEAVE();
>>>> + return rc;
>>>> +}
>>>> +/****************************************************************************
>>>>
>>>> + Name : recoverSubscriber
>>>> +
>>>> + Description : This routine sends subscribe messages to NTF
>>>> Server in
>>>> + order to reintroduce this subscriber to NTF Server
>>>> +
>>>> + Arguments : client_hdl* [IN]: client handle
>>>> + subscriber_hdl* [IN]: subscriber handle
>>>> +
>>>> + Return Values : SA_AIS_OK if success, others are failure
>>>> +
>>>> + Notes : None
>>>> +******************************************************************************/
>>>>
>>>> +SaAisErrorT recoverSubscriber(ntfa_client_hdl_rec_t* client_hdl,
>>>> + ntfa_subscriber_list_t*
>>>> subscriber_hdl) {
>>>> + uint32_t mds_rc;
>>>> + SaAisErrorT rc = SA_AIS_OK;
>>>> + ntfsv_msg_t i_msg, *o_msg = NULL;
>>>> + ntfsv_subscribe_req_t *send_param;
>>>> +
>>>> + TRACE_ENTER();
>>>> +
>>>> + /* Make sure the filters have not been deleted */
>>>> + if (subscriber_hdl->filters.alarm_filter == NULL &&
>>>> + subscriber_hdl->filters.att_ch_filter == NULL &&
>>>> + subscriber_hdl->filters.obj_cr_del_filter == NULL &&
>>>> + subscriber_hdl->filters.sec_al_filter == NULL &&
>>>> + subscriber_hdl->filters.sta_ch_filter == NULL)
>>>> + return SA_AIS_ERR_BAD_HANDLE;
>>>> +
>>>> + memset(&i_msg, 0, sizeof(ntfsv_msg_t));
>>>> + i_msg.type = NTFSV_NTFA_API_MSG;
>>>> + i_msg.info.api_info.type = NTFSV_SUBSCRIBE_REQ;
>>>> + send_param = &i_msg.info.api_info.param.subscribe;
>>>> +
>>>> + send_param->client_id = client_hdl->ntfs_client_id;
>>>> + send_param->subscriptionId =
>>>> subscriber_hdl->subscriberListSubscriptionId;
>>>> + send_param->f_rec = subscriber_hdl->filters;
>>>> +
>>>> + mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &i_msg, &o_msg,
>>>> NTFS_WAIT_TIME);
>>>> +
>>>> + switch (mds_rc) {
>>>> + case NCSCC_RC_SUCCESS:
>>>> + break;
>>>> + case NCSCC_RC_REQ_TIMOUT:
>>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>>> + goto done;
>>>> + default:
>>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>>> + goto done;
>>>> + }
>>>> +
>>>> + osafassert(o_msg != NULL);
>>>> +
>>>> + if ((rc = o_msg->info.api_resp_info.rc) != SA_AIS_OK) {
>>>> + TRACE("o_msg->info.api_resp_info.rc:%u",
>>>> o_msg->info.api_resp_info.rc);
>>>> + rc = SA_AIS_ERR_BAD_HANDLE;
>>>> + goto done;
>>>> + }
>>>> +
>>>> + if (o_msg->info.api_resp_info.type != NTFSV_SUBSCRIBE_RSP) {
>>>> + TRACE("msg type (%d) failed",
>>>> (int)o_msg->info.api_resp_info.type);
>>>> + rc = SA_AIS_ERR_LIBRARY;
>>>> + goto done;
>>>> + }
>>>> + TRACE("Recover subscriber successfully");
>>>> +done:
>>>> + if (o_msg)
>>>> + ntfa_msg_destroy(o_msg);
>>>> +
>>>> + TRACE_LEAVE();
>>>> + return rc;
>>>> +}
>>>> +/****************************************************************************
>>>>
>>>> + Name : recoverClient
>>>> +
>>>> + Description : In NTF-A.03.01, section 2.1.2. A client can be
>>>> producer,
>>>> + subscriber, and reader at the same time.
>>>> + This routine will recovery:
>>>> + (1) client_id
>>>> + (2) Then it will continue the recovery if this client also
>>>> + has instance of subscriber or reader.
>>>> + Arguments : client_hdl* [IN]: client handle
>>>> +
>>>> + Return Values : SA_AIS_OK if success, others are failure
>>>> +
>>>> + Notes : None
>>>> +******************************************************************************/
>>>>
>>>> +SaAisErrorT recoverClient(ntfa_client_hdl_rec_t *client_hdl) {
>>>> +
>>>> + TRACE_ENTER();
>>>> + SaAisErrorT rc = SA_AIS_OK;
>>>> +
>>>> + osafassert(client_hdl);
>>>> +
>>>> + if ((rc = reinitializeClient(client_hdl)) == SA_AIS_OK) {
>>>> + /* Restore reader */
>>>> + ntfa_reader_hdl_rec_t* reader_hdl = client_hdl->reader_list;
>>>> + while (reader_hdl != NULL && rc == SA_AIS_OK) {
>>>> + rc = recoverReader(client_hdl, reader_hdl);
>>>> + reader_hdl = reader_hdl->next;
>>>> + }
>>>> + if (rc != SA_AIS_OK) {
>>>> + TRACE("Failed to restore reader (readerId:%d)",
>>>> + reader_hdl->reader_id);
>>>> + goto done;
>>>> + }
>>>> + /* Restore subscriber */
>>>> + ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
>>>> + while (subscriber_hdl != NULL && rc == SA_AIS_OK) {
>>>> + if (client_hdl->local_hdl ==
>>>> subscriber_hdl->subscriberListNtfHandle)
>>>> + rc = recoverSubscriber(client_hdl, subscriber_hdl);
>>>> + subscriber_hdl = subscriber_hdl->next;
>>>> + }
>>>> + if (rc != SA_AIS_OK) {
>>>> + TRACE("Failed to restore subscriber (subscriptionId:%d)",
>>>> + subscriber_hdl->subscriberListSubscriptionId);
>>>> + goto done;
>>>> + }
>>>> + client_hdl->valid = true;
>>>> + } else {
>>>> + TRACE("Failed to restore client (id:%d)",
>>>> client_hdl->ntfs_client_id);
>>>> + goto done;
>>>> + }
>>>> +
>>>> +done:
>>>> + TRACE_LEAVE();
>>>> + return rc;
>>>> +}
>>>> /***************************************************************************
>>>>
>>>> * 8.4.1
>>>> *
>>>> @@ -923,15 +1213,14 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>>> version->releaseCode = NTF_RELEASE_CODE;
>>>> version->majorVersion = NTF_MAJOR_VERSION;
>>>> version->minorVersion = NTF_MINOR_VERSION;
>>>> - ntfa_shutdown();
>>>> + ntfa_shutdown(false);
>>>> rc = SA_AIS_ERR_VERSION;
>>>> goto done;
>>>> }
>>>>
>>>> - if (!ntfa_cb.ntfs_up) {
>>>> - ntfa_shutdown();
>>>> - TRACE("NTFS server is down");
>>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>>> + /* Check NTF server availability */
>>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>>>> + ntfa_shutdown(false);
>>>> goto done;
>>>> }
>>>>
>>>> @@ -977,7 +1266,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>>> rc = SA_AIS_ERR_NO_MEMORY;
>>>> goto err;
>>>> }
>>>> -
>>>> + ntfa_hdl_rec->version = *version;
>>>> /* pass the handle value to the appl */
>>>> if (SA_AIS_OK == rc)
>>>> *ntfHandle = ntfa_hdl_rec->local_hdl;
>>>> @@ -990,7 +1279,7 @@ SaAisErrorT saNtfInitialize(SaNtfHandleT
>>>>
>>>> if (rc != SA_AIS_OK) {
>>>> TRACE_2("NTFA INIT FAILED\n");
>>>> - ntfa_shutdown();
>>>> + ntfa_shutdown(false);
>>>> }
>>>>
>>>> done:
>>>> @@ -1101,6 +1390,28 @@ SaAisErrorT saNtfDispatch(SaNtfHandleT n
>>>> goto done;
>>>> }
>>>>
>>>> + /* Need to check NTF server availability here
>>>> + * Because if the notificationCallback just comes after MDS_DOWN
>>>> + * it will go to recovery then result in failure due to no director
>>>> + */
>>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>>>> + ncshm_give_hdl(ntfHandle);
>>>> + goto done;
>>>> + }
>>>> +
>>>> + if (!hdl_rec->valid) {
>>>> + /* recovery */
>>>> + if ((rc = recoverClient(hdl_rec)) != SA_AIS_OK) {
>>>> + if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>>> + ncshm_give_hdl(ntfHandle);
>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, hdl_rec);
>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_shutdown(false);
>>>> + goto done;
>>>> + }
>>>> + }
>>>> + }
>>>> if ((rc = ntfa_hdl_cbk_dispatch(&ntfa_cb, hdl_rec,
>>>> dispatchFlags)) != SA_AIS_OK)
>>>> TRACE("NTFA_DISPATCH_FAILURE");
>>>>
>>>> @@ -1145,6 +1456,13 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>>>
>>>> TRACE_ENTER();
>>>>
>>>> + /* Check NTF server availability */
>>>> + if (ntfa_cb.ntfa_ntfsv_state == NTFA_NTFSV_NO_ACTIVE) {
>>>> + TRACE("NTFS server is temporarily unavailable");
>>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>>> + goto done;
>>>> + }
>>>> +
>>>> /* retrieve hdl rec */
>>>> hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA, ntfHandle);
>>>> if (hdl_rec == NULL) {
>>>> @@ -1152,43 +1470,37 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>>> rc = SA_AIS_ERR_BAD_HANDLE;
>>>> goto done;
>>>> }
>>>> -
>>>> - /* Check Whether NTFS is up or not */
>>>> - if (!ntfa_cb.ntfs_up) {
>>>> - TRACE("NTFS down");
>>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>>> - goto done_give_hdl;
>>>> + if (hdl_rec->valid) {
>>>> + /** populate & send the finalize message
>>>> + ** and make sure the finalize from the server
>>>> + ** end returned before deleting the local records.
>>>> + **/
>>>> + memset(&msg, 0, sizeof(ntfsv_msg_t));
>>>> + msg.type = NTFSV_NTFA_API_MSG;
>>>> + msg.info.api_info.type = NTFSV_FINALIZE_REQ;
>>>> + msg.info.api_info.param.finalize.client_id =
>>>> hdl_rec->ntfs_client_id;
>>>> +
>>>> + mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>>>> NTFS_WAIT_TIME);
>>>> + switch (mds_rc) {
>>>> + case NCSCC_RC_SUCCESS:
>>>> + break;
>>>> + case NCSCC_RC_REQ_TIMOUT:
>>>> + rc = SA_AIS_ERR_TIMEOUT;
>>>> + TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>>> + goto done_give_hdl;
>>>> + default:
>>>> + TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>>> + rc = SA_AIS_ERR_NO_RESOURCES;
>>>> + goto done_give_hdl;
>>>> + }
>>>> +
>>>> + if (o_msg != NULL) {
>>>> + rc = o_msg->info.api_resp_info.rc;
>>>> + ntfa_msg_destroy(o_msg);
>>>> + } else
>>>> + rc = SA_AIS_ERR_NO_RESOURCES;
>>>> }
>>>>
>>>> - /** populate & send the finalize message
>>>> - ** and make sure the finalize from the server
>>>> - ** end returned before deleting the local records.
>>>> - **/
>>>> - memset(&msg, 0, sizeof(ntfsv_msg_t));
>>>> - msg.type = NTFSV_NTFA_API_MSG;
>>>> - msg.info.api_info.type = NTFSV_FINALIZE_REQ;
>>>> - msg.info.api_info.param.finalize.client_id =
>>>> hdl_rec->ntfs_client_id;
>>>> -
>>>> - mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg,
>>>> NTFS_WAIT_TIME);
>>>> - switch (mds_rc) {
>>>> - case NCSCC_RC_SUCCESS:
>>>> - break;
>>>> - case NCSCC_RC_REQ_TIMOUT:
>>>> - rc = SA_AIS_ERR_TIMEOUT;
>>>> - TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>>> - goto done_give_hdl;
>>>> - default:
>>>> - TRACE("ntfa_mds_msg_sync_send FAILED: %u", rc);
>>>> - rc = SA_AIS_ERR_NO_RESOURCES;
>>>> - goto done_give_hdl;
>>>> - }
>>>> -
>>>> - if (o_msg != NULL) {
>>>> - rc = o_msg->info.api_resp_info.rc;
>>>> - ntfa_msg_destroy(o_msg);
>>>> - } else
>>>> - rc = SA_AIS_ERR_NO_RESOURCES;
>>>> -
>>>> if (rc == SA_AIS_OK) {
>>>> /** delete the hdl rec
>>>> ** including all resources allocated by client if MDS send is
>>>> @@ -1205,7 +1517,7 @@ SaAisErrorT saNtfFinalize(SaNtfHandleT n
>>>> ncshm_give_hdl(ntfHandle);
>>>>
>>>> if (rc == SA_AIS_OK) {
>>>> - rc = ntfa_shutdown();
>>>> + rc = ntfa_shutdown(false);
>>>> if (rc != NCSCC_RC_SUCCESS)
>>>> TRACE_1("ntfa_shutdown failed");
>>>> }
>>>> @@ -1349,7 +1661,7 @@ SaAisErrorT saNtfNotificationFree(SaNtfN
>>>> }
>>>>
>>>> /* free the resources allocated by
>>>> saNtf<ntfType>NotificationAllocate */
>>>> - ntfa_hdl_rec_destructor(notification_hdl_rec);
>>>> + ntfa_notification_destructor(notification_hdl_rec);
>>>>
>>>> /** Delete the resources related to the notificationHandle &
>>>> * remove reference in the client.
>>>> @@ -1419,6 +1731,24 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>>>> }
>>>> osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>
>>>> + /* Check NTF server availability */
>>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>>> + goto done_give_hdls;
>>>> +
>>>> + /* Recover if this is invalid handle */
>>>> + if (!client_rec->valid) {
>>>> + if ((rc = recoverClient(client_rec)) != SA_AIS_OK) {
>>>> + ncshm_give_hdl(client_handle);
>>>> + ncshm_give_hdl(notificationHandle);
>>>> + if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_rec);
>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_shutdown(false);
>>>> + }
>>>> + goto err_free;
>>>> + }
>>>> + }
>>>> /**
>>>> ** Populate a sync MDS message
>>>> **/
>>>> @@ -1481,13 +1811,6 @@ SaAisErrorT saNtfNotificationSend(SaNtfN
>>>> goto done_give_hdls;
>>>> }
>>>>
>>>> - /* Check whether NTFS is up or not */
>>>> - if (!ntfa_cb.ntfs_up) {
>>>> - TRACE("NTFS down");
>>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>>> - goto done_give_hdls;
>>>> - }
>>>> -
>>>> /* Send a sync MDS message to obtain a notification id */
>>>> mds_rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>>> switch (mds_rc) {
>>>> @@ -1596,7 +1919,8 @@ static SaNtfHandleT ntfHandleGet(SaNtfS
>>>> return ntfHandle;
>>>> }
>>>>
>>>> -static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
>>>> SaNtfSubscriptionIdT subscriptionId)
>>>> +static SaAisErrorT subscriptionListAdd(SaNtfHandleT ntfHandle,
>>>> SaNtfSubscriptionIdT subscriptionId,
>>>> + ntfsv_filter_ptrs_t filters)
>>>> {
>>>> SaAisErrorT rc = SA_AIS_OK;
>>>> ntfa_subscriber_list_t* ntfSubscriberList;
>>>> @@ -1610,7 +1934,9 @@ static SaAisErrorT subscriptionListAdd(S
>>>> /* Add ntfHandle and subscriptionId into list */
>>>> ntfSubscriberList->subscriberListNtfHandle = ntfHandle;
>>>> ntfSubscriberList->subscriberListSubscriptionId = subscriptionId;
>>>> -
>>>> + memset(&ntfSubscriberList->filters, 0,
>>>> sizeof(ntfsv_filter_ptrs_t));
>>>> + ntfa_copy_ntf_filter_ptrs(&ntfSubscriberList->filters, &filters);
>>>> +
>>>> osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>> if (NULL == subscriberNoList) {
>>>> subscriberNoList = ntfSubscriberList;
>>>> @@ -1648,6 +1974,7 @@ static void subscriberListItemRemove(SaN
>>>> subscriberNoList = NULL;
>>>> }
>>>> TRACE_1("REMOVE: listPtr->SubscriptionId %d",
>>>> listPtr->subscriberListSubscriptionId);
>>>> + ntfa_del_ntf_filter_ptrs(&listPtr->filters);
>>>> free(listPtr);
>>>> osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>> }
>>>> @@ -1666,20 +1993,33 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>>> ntfsv_msg_t msg, *o_msg = NULL;
>>>> ntfsv_subscribe_req_t *send_param;
>>>> uint32_t timeout = NTFS_WAIT_TIME;
>>>> -
>>>> + bool recovery_failed = false;
>>>> TRACE_ENTER();
>>>> +
>>>> rc = getFilters(notificationFilterHandles, &filters,
>>>> &ntfHandle, &client_hdl_rec);
>>>> if (rc != SA_AIS_OK) {
>>>> TRACE("getFilters failed");
>>>> goto done;
>>>> }
>>>>
>>>> + /* Check NTF server availability */
>>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>>> + goto done;
>>>> +
>>>> + /* recovery */
>>>> + if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
>>>> + if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>>> + recovery_failed = true;
>>>> + goto done;
>>>> + }
>>>> + }
>>>> +
>>>> tmpHandle = ntfHandleGet(subscriptionId);
>>>> if (tmpHandle != 0) {
>>>> rc = SA_AIS_ERR_EXIST;
>>>> goto done;
>>>> }
>>>> - rc = subscriptionListAdd(ntfHandle, subscriptionId);
>>>> + rc = subscriptionListAdd(ntfHandle, subscriptionId, filters);
>>>> if (rc != SA_AIS_OK) {
>>>> goto done;
>>>> }
>>>> @@ -1693,29 +2033,24 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>>> send_param->client_id = client_hdl_rec->ntfs_client_id;
>>>> send_param->subscriptionId = subscriptionId;
>>>> send_param->f_rec = filters;
>>>> - /* Check whether NTFS is up or not */
>>>> - if (ntfa_cb.ntfs_up) {
>>>> - uint32_t rv;
>>>> - /* Send a sync MDS message to obtain a log stream id */
>>>> - rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>>> - if (rv == NCSCC_RC_SUCCESS) {
>>>> - osafassert(o_msg != NULL);
>>>> - if (SA_AIS_OK == o_msg->info.api_resp_info.rc) {
>>>> - TRACE_1("subscriptionId from server %u",
>>>> - o_msg->info.api_resp_info.param.subscribe_rsp.subscriptionId);
>>>> - } else {
>>>> - rc = o_msg->info.api_resp_info.rc;
>>>> - TRACE("Bad return status!!! rc = %d", rc);
>>>> - }
>>>> +
>>>> + uint32_t rv;
>>>> + /* Send a sync MDS message to obtain a log stream id */
>>>> + rv = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>>> + if (rv == NCSCC_RC_SUCCESS) {
>>>> + osafassert(o_msg != NULL);
>>>> + if (SA_AIS_OK == o_msg->info.api_resp_info.rc) {
>>>> + TRACE_1("subscriptionId from server %u",
>>>> + o_msg->info.api_resp_info.param.subscribe_rsp.subscriptionId);
>>>> } else {
>>>> - if(rv == NCSCC_RC_INVALID_INPUT)
>>>> - rc = SA_AIS_ERR_INVALID_PARAM;
>>>> - else
>>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>>> + rc = o_msg->info.api_resp_info.rc;
>>>> + TRACE("Bad return status!!! rc = %d", rc);
>>>> }
>>>> } else {
>>>> - TRACE_1("NTFS down");
>>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>>> + if(rv == NCSCC_RC_INVALID_INPUT)
>>>> + rc = SA_AIS_ERR_INVALID_PARAM;
>>>> + else
>>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>>> }
>>>>
>>>> if (rc != SA_AIS_OK) {
>>>> @@ -1724,6 +2059,7 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>>> if (o_msg)
>>>> ntfa_msg_destroy(o_msg);
>>>> done:
>>>> +
>>>> ncshm_give_hdl(ntfHandle);
>>>> if (notificationFilterHandles) {
>>>> if (notificationFilterHandles->attributeChangeFilterHandle)
>>>> @@ -1737,6 +2073,13 @@ SaAisErrorT saNtfNotificationSubscribe(c
>>>> if (notificationFilterHandles->alarmFilterHandle)
>>>> ncshm_give_hdl(notificationFilterHandles->alarmFilterHandle);
>>>> }
>>>> + if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_shutdown(false);
>>>> + }
>>>> +
>>>> TRACE_LEAVE();
>>>> return rc;
>>>> }
>>>> @@ -1995,6 +2338,7 @@ SaAisErrorT saNtfSecurityAlarmNotificati
>>>> rc = SA_AIS_ERR_INVALID_PARAM;
>>>> goto done;
>>>> }
>>>> +
>>>> /* retrieve hdl rec */
>>>> hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA, ntfHandle);
>>>> if (hdl_rec == NULL) {
>>>> @@ -2058,6 +2402,7 @@ SaAisErrorT saNtfPtrValAllocate(SaNtfNot
>>>> rc = SA_AIS_ERR_INVALID_PARAM;
>>>> goto done;
>>>> }
>>>> +
>>>> notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>>>> notificationHandle);
>>>> if (notification_hdl_rec == NULL) {
>>>> TRACE("ncshm_take_hdl notificationHandle failed");
>>>> @@ -2098,6 +2443,7 @@ SaAisErrorT saNtfArrayValAllocate(SaNtfN
>>>> rc = SA_AIS_ERR_INVALID_PARAM;
>>>> goto done;
>>>> }
>>>> +
>>>> notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>>>> notificationHandle);
>>>> if (notification_hdl_rec == NULL) {
>>>> TRACE("ncshm_take_hdl notificationHandle failed");
>>>> @@ -2228,6 +2574,7 @@ SaAisErrorT saNtfArrayValGet(SaNtfNotifi
>>>> rc = SA_AIS_ERR_INVALID_PARAM;
>>>> goto done;
>>>> }
>>>> +
>>>> notification_hdl_rec = ncshm_take_hdl(NCS_SERVICE_ID_NTFA,
>>>> notificationHandle);
>>>> if (notification_hdl_rec == NULL) {
>>>> TRACE("ncshm_take_hdl notificationHandle failed");
>>>> @@ -2695,7 +3042,7 @@ SaAisErrorT saNtfNotificationFilterFree(
>>>> }
>>>>
>>>> /* free the resources allocated by saNtf<ntfType>FilterAllocate */
>>>> - ntfa_filter_hdl_rec_destructor(filter_hdl_rec);
>>>> + ntfa_filter_destructor(filter_hdl_rec);
>>>>
>>>> /** Delete the resources related to the notificationFilterHandle &
>>>> * remove reference in the client.
>>>> @@ -2729,7 +3076,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>> TRACE_ENTER();
>>>> SaAisErrorT rc = SA_AIS_ERR_NOT_EXIST;
>>>> SaNtfHandleT ntfHandle;
>>>> -
>>>> ntfa_client_hdl_rec_t *client_hdl_rec;
>>>>
>>>> ntfsv_msg_t msg, *o_msg = NULL;
>>>> @@ -2737,7 +3083,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>> ntfsv_unsubscribe_req_t *send_param;
>>>> uint32_t timeout = NTFS_WAIT_TIME;
>>>>
>>>> -
>>>> ntfHandle = ntfHandleGet(subscriptionId);
>>>> if (ntfHandle == 0) {
>>>> TRACE_1("ntfHandleGet failed, subscription not exist");
>>>> @@ -2753,9 +3098,16 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>> goto done;
>>>> }
>>>>
>>>> - /**
>>>> - ** Populate a sync MDS message
>>>> - **/
>>>> + /* Check NTF server availability */
>>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>>> [Praveen] Here in both unavailability and down case, API will return
>>> without clean up of subscriber information. But README in patch 3 tals
>>> differently for the two cases.
>>> I think handling like saNtfFinalize() way needs to be done here.
>> [Minh]: I will have to check the history commit, to see if any reason
>> that agent doesn't silently release subscriber's resource here as
>> described in README. But it looks to me a bug, can I get back to you later?
> [Praveen] This needs to be corrected otherwise the same application will
> not be able to reuse old subscription_id. It will get ERR_EXIST.
>>>
>>>> + ncshm_give_hdl(ntfHandle);
>>>> + goto done;
>>>> + }
>>>> +
>>>> + if (client_hdl_rec->valid) {
>>>> + /**
>>>> + ** Populate a sync MDS message
>>>> + **/
>>>> memset(&msg, 0, sizeof(ntfsv_msg_t));
>>>> msg.type = NTFSV_NTFA_API_MSG;
>>>> msg.info.api_info.type = NTFSV_UNSUBSCRIBE_REQ;
>>>> @@ -2764,13 +3116,6 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>> send_param->client_id = client_hdl_rec->ntfs_client_id;
>>>> send_param->subscriptionId = subscriptionId;
>>>>
>>>> - /* Check whether NTFS is up or not */
>>>> - if (!ntfa_cb.ntfs_up) {
>>>> - TRACE_1("NTFS down");
>>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>>> - goto done_give_hdl;
>>>> - }
>>>> -
>>>> /* Send a sync MDS message to obtain a log stream id */
>>>> rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>>> if (rc != NCSCC_RC_SUCCESS) {
>>>> @@ -2785,6 +3130,7 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>> TRACE_1("Bad return status! rc = %d", rc);
>>>> goto done_give_hdl;
>>>> }
>>>> + }
>>>> subscriberListItemRemove(subscriptionId);
>>>>
>>>> /*Remove msg for subscriptionId from mailbox*/
>>>> @@ -2815,6 +3161,19 @@ SaAisErrorT saNtfNotificationUnsubscribe
>>>> done_give_hdl:
>>>> if (o_msg)
>>>> ntfa_msg_destroy(o_msg);
>>>> +
>>>> + if (!client_hdl_rec->valid) {
>>>> + if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>>> + if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>>> + ncshm_give_hdl(ntfHandle);
>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_shutdown(false);
>>>> + goto done;
>>>> + }
>>>> + }
>>>> + }
>>>> ncshm_give_hdl(ntfHandle);
>>>> done:
>>>> TRACE_LEAVE();
>>>> @@ -2838,6 +3197,7 @@ SaAisErrorT saNtfNotificationReadInitial
>>>> ntfsv_reader_init_req_2_t *send_param;
>>>> uint32_t timeout = NTFS_WAIT_TIME;
>>>>
>>>> + bool recovery_failed = false;
>>>> TRACE_ENTER();
>>>> if (notificationFilterHandles == NULL || readHandle == NULL) {
>>>> rc = SA_AIS_ERR_INVALID_PARAM;
>>>> @@ -2872,6 +3232,18 @@ SaAisErrorT saNtfNotificationReadInitial
>>>> goto done_give_client_hdl;
>>>> }
>>>>
>>>> + /* Check NTF server availability */
>>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>>> + goto done_give_client_hdl;
>>>> +
>>>> + /* recovery */
>>>> + if (client_hdl_rec != NULL && !client_hdl_rec->valid) {
>>>> + if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>>> + recovery_failed = true;
>>>> + goto done_give_client_hdl;
>>>> + }
>>>> + }
>>>> +
>>>> /**
>>>> ** Populate a sync MDS message
>>>> **/
>>>> @@ -2884,13 +3256,6 @@ SaAisErrorT saNtfNotificationReadInitial
>>>> send_param->head.searchCriteria = searchCriteria;
>>>> send_param->f_rec = filters;
>>>>
>>>> - /* Check whether NTFS is up or not */
>>>> - if (!ntfa_cb.ntfs_up) {
>>>> - TRACE("NTFS down");
>>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>>> - goto done_give_client_hdl;
>>>> - }
>>>> -
>>>> /* Send a sync MDS message to obtain a log stream id */
>>>> rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>>> if (rc != NCSCC_RC_SUCCESS) {
>>>> @@ -2925,12 +3290,16 @@ SaAisErrorT saNtfNotificationReadInitial
>>>> reader_hdl_rec->ntfHandle = ntfHandle;
>>>> /* Store the readerId returned from server */
>>>> reader_hdl_rec->reader_id =
>>>> o_msg->info.api_resp_info.param.reader_init_rsp.readerId;
>>>> + memset(&reader_hdl_rec->filters, 0, sizeof(ntfsv_filter_ptrs_t));
>>>> + ntfa_copy_ntf_filter_ptrs(&reader_hdl_rec->filters, &filters);
>>>> + reader_hdl_rec->searchCriteria = searchCriteria;
>>>> /** UnLock ntfa_CB **/
>>>> osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>>
>>>> done_give_client_hdl:
>>>> if (o_msg)
>>>> ntfa_msg_destroy(o_msg);
>>>> +
>>>> if (client_hdl_rec)
>>>> ncshm_give_hdl(client_hdl_rec->local_hdl);
>>>> if (notificationFilterHandles) {
>>>> @@ -2947,6 +3316,12 @@ done_give_client_hdl:
>>>> }
>>>>
>>>> ncshm_give_hdl(notificationFilterHandles->alarmFilterHandle);
>>>> + if (recovery_failed && rc == SA_AIS_ERR_BAD_HANDLE) {
>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_shutdown(false);
>>>> + }
>>>> done:
>>>> TRACE_LEAVE();
>>>> return rc;
>>>> @@ -2983,42 +3358,46 @@ SaAisErrorT saNtfNotificationReadFinaliz
>>>> }
>>>> TRACE_1("reader_hdl_rec = %u", reader_hdl_rec->reader_hdl);
>>>>
>>>> - /**
>>>> - ** Populate a sync MDS message
>>>> - **/
>>>> - memset(&msg, 0, sizeof(ntfsv_msg_t));
>>>> - msg.type = NTFSV_NTFA_API_MSG;
>>>> - msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
>>>> - send_param = &msg.info.api_info.param.reader_finalize;
>>>> - send_param->client_id = client_hdl_rec->ntfs_client_id;
>>>> - send_param->readerId = reader_hdl_rec->reader_id;
>>>> -
>>>> - /* Check whether NTFS is up or not */
>>>> - if (!ntfa_cb.ntfs_up) {
>>>> - TRACE("NTFS down");
>>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>>> - goto done_give_hdls;
>>>> + /* Check NTF server availability */
>>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK) {
>>> [Praveen] Same comment here as in Unsubscribe().
>>> [Minh] As above in Unsubscribed()
>
>>>> + ncshm_give_hdl(client_hdl_rec->local_hdl);
>>>> + ncshm_give_hdl(readhandle);
>>>> + goto done;
>>>> }
>>>>
>>>> - /* Send a sync MDS message */
>>>> - rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>>> - if (rc != NCSCC_RC_SUCCESS) {
>>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>>> - goto done_give_hdls;
>>>> + if (client_hdl_rec->valid) {
>>>> + /**
>>>> + ** Populate a sync MDS message
>>>> + **/
>>>> + memset(&msg, 0, sizeof(ntfsv_msg_t));
>>>> + msg.type = NTFSV_NTFA_API_MSG;
>>>> + msg.info.api_info.type = NTFSV_READER_FINALIZE_REQ;
>>>> + send_param = &msg.info.api_info.param.reader_finalize;
>>>> + send_param->client_id = client_hdl_rec->ntfs_client_id;
>>>> + send_param->readerId = reader_hdl_rec->reader_id;
>>>> +
>>>> +
>>>> + /* Send a sync MDS message */
>>>> + rc = ntfa_mds_msg_sync_send(&ntfa_cb, &msg, &o_msg, timeout);
>>>> + if (rc != NCSCC_RC_SUCCESS) {
>>>> + rc = SA_AIS_ERR_TRY_AGAIN;
>>>> + goto done_give_hdls;
>>>> + }
>>>> +
>>>> + osafassert(o_msg != NULL);
>>>> + if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
>>>> + rc = o_msg->info.api_resp_info.rc;
>>>> + TRACE("Bad return status!!! rc = %d", rc);
>>>> + goto done_give_hdls;
>>>> + }
>>>> +
>>>> + if (o_msg->info.api_resp_info.type !=
>>>> NTFSV_READER_FINALIZE_RSP) {
>>>> + TRACE("msg type (%d) failed",
>>>> (int)o_msg->info.api_resp_info.type);
>>>> + rc = SA_AIS_ERR_LIBRARY;
>>>> + goto done_give_hdls;
>>>> + }
>>>> }
>>>>
>>>> - osafassert(o_msg != NULL);
>>>> - if (SA_AIS_OK != o_msg->info.api_resp_info.rc) {
>>>> - rc = o_msg->info.api_resp_info.rc;
>>>> - TRACE("Bad return status!!! rc = %d", rc);
>>>> - goto done_give_hdls;
>>>> - }
>>>> -
>>>> - if (o_msg->info.api_resp_info.type != NTFSV_READER_FINALIZE_RSP) {
>>>> - TRACE("msg type (%d) failed",
>>>> (int)o_msg->info.api_resp_info.type);
>>>> - rc = SA_AIS_ERR_LIBRARY;
>>>> - goto done_give_hdls;
>>>> - }
>>>> osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>> oas_rc = ntfa_reader_hdl_rec_del(&client_hdl_rec->reader_list,
>>>> reader_hdl_rec);
>>>> if (oas_rc != NCSCC_RC_SUCCESS) {
>>>> @@ -3029,6 +3408,20 @@ SaAisErrorT saNtfNotificationReadFinaliz
>>>> done_give_hdls:
>>>> if (o_msg)
>>>> ntfa_msg_destroy(o_msg);
>>>> +
>>>> + if (!client_hdl_rec->valid) {
>>>> + if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>>> + if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>>> + ncshm_give_hdl(client_hdl_rec->local_hdl);
>>>> + ncshm_give_hdl(readhandle);
>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_shutdown(false);
>>>> + goto done;
>>>> + }
>>>> + }
>>>> + }
>>>> ncshm_give_hdl(client_hdl_rec->local_hdl);
>>>> done_give_read_hdl:
>>>> ncshm_give_hdl(readhandle);
>>>> @@ -3082,6 +3475,23 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>>>> }
>>>> TRACE_1("reader_hdl_rec = %u", reader_hdl_rec->reader_hdl);
>>>>
>>>> + /* Check NTF server availability */
>>>> + if ((rc = checkNtfServerState()) != SA_AIS_OK)
>>>> + goto done_give_hdls;
>>>> +
>>>> + if (!client_hdl_rec->valid) {
>>>> + if ((rc = recoverClient(client_hdl_rec)) != SA_AIS_OK) {
>>>> + ncshm_give_hdl(client_hdl_rec->local_hdl);
>>>> + ncshm_give_hdl(readHandle);
>>>> + if (rc == SA_AIS_ERR_BAD_HANDLE) {
>>>> + osafassert(pthread_mutex_lock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_hdl_rec_force_del(&ntfa_cb.client_list, client_hdl_rec);
>>>> + osafassert(pthread_mutex_unlock(&ntfa_cb.cb_lock) == 0);
>>>> + ntfa_shutdown(false);
>>>> + }
>>>> + goto done;
>>>> + }
>>>> + }
>>>> /**
>>>> ** Populate a sync MDS message
>>>> **/
>>>> @@ -3092,12 +3502,6 @@ SaAisErrorT saNtfNotificationReadNext(Sa
>>>> send_param->client_id = client_hdl_rec->ntfs_client_id;
>>>> send_param->readerId = reader_hdl_rec->reader_id;
>>>> send_param->searchDirection = searchDirection;
>>>> - /* Check whether NTFS is up or not */
>>>> - if (!ntfa_cb.ntfs_up) {
>>>> - TRACE("NTFS down");
>>>> - rc = SA_AIS_ERR_TRY_AGAIN;
>>>> - goto done_give_hdls;
>>>> - }
>>>>
>>>> do {
>>>> /* Send a sync MDS message */
>>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>>> b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>>> --- a/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_mds.c
>>>> @@ -369,7 +369,8 @@ static uint32_t ntfa_mds_svc_evt(struct
>>>> TRACE("NTFS down");
>>>> pthread_mutex_lock(&ntfa_cb.cb_lock);
>>>> memset(&ntfa_cb.ntfs_mds_dest, 0, sizeof(MDS_DEST));
>>>> - ntfa_cb.ntfs_up = 0;
>>>> + ntfa_update_ntfsv_state(mds_cb_info->info.svc_evt.i_change ==
>>>> NCSMDS_NO_ACTIVE ?
>>>> + NTFA_NTFSV_NO_ACTIVE : NTFA_NTFSV_DOWN);
>>>> pthread_mutex_unlock(&ntfa_cb.cb_lock);
>>>> }
>>>> break;
>>>> @@ -382,7 +383,8 @@ static uint32_t ntfa_mds_svc_evt(struct
>>>> TRACE_2("MSG from NTFS NCSMDS_NEW_ACTIVE/UP");
>>>> pthread_mutex_lock(&ntfa_cb.cb_lock);
>>>> ntfa_cb.ntfs_mds_dest = mds_cb_info->info.svc_evt.i_dest;
>>>> - ntfa_cb.ntfs_up = 1;
>>>> + ntfa_update_ntfsv_state(mds_cb_info->info.svc_evt.i_change ==
>>>> NCSMDS_NEW_ACTIVE ?
>>>> + NTFA_NTFSV_NEW_ACTIVE : NTFA_NTFSV_UP);
>>>> if (ntfa_cb.ntfs_sync_awaited) {
>>>> /* signal waiting thread */
>>>> m_NCS_SEL_OBJ_IND(&ntfa_cb.ntfs_sync_sel);
>>>> diff --git a/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>>> b/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>>> --- a/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>>> +++ b/osaf/libs/agents/saf/ntfa/ntfa_util.c
>>>> @@ -49,7 +49,7 @@ static unsigned int ntfa_create(void)
>>>> }
>>>>
>>>> /* Block and wait for indication from MDS meaning NTFS is up */
>>>> - osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_sync_sel), 30000);
>>>> + osaf_poll_one_fd(m_GET_FD_FROM_SEL_OBJ(ntfa_cb.ntfs_sync_sel), 10000);
>>> [Praveen] Did not get why it is changed to 10 seconds.
>>> [Minh] I think 30 seconds is just too long
>>>>
>>>> pthread_mutex_lock(&ntfa_cb.cb_lock);
>>>> ntfa_cb.ntfs_sync_awaited = 0;
>>>> @@ -108,7 +108,25 @@ static bool ntfa_clear_mbx(NCSCONTEXT ar
>>>> }
>>>> return true;
>>>> }
>>>> +/****************************************************************************
>>>>
>>>> + Name : ntfa_notification_list_del
>>>>
>>>> + Description : This routine free element allocated in list of
>>>> notification
>>>> +
>>>> + Arguments : pointer to the list of notification records anchor.
>>>> +
>>>> + Return Values : None
>>>> +
>>>> + Notes :
>>>> +******************************************************************************/
>>>>
>>>> +static void ntfa_notification_list_del(ntfa_notification_hdl_rec_t
>>>> **plstr_hdl)
>>>> +{
>>>> + ntfa_notification_hdl_rec_t *lstr_hdl = *plstr_hdl;
>>>> + while (lstr_hdl != NULL) {
>>>> + ntfa_notification_destructor(lstr_hdl);
>>>> + lstr_hdl = lstr_hdl->next;
>>>> + }
>>>> +}
>>>> /****************************************************************************
>>>>
>>>> Name : ntfa_notification_hdl_rec_list_del
>>>>
>>>> @@ -130,6 +148,70 @@ static void ntfa_notification_hdl_rec_li
>>>> lstr_hdl = NULL;
>>>> }
>>>> }
>>>> +/****************************************************************************
>>>>
>>>> + Name : ntfa_filter_list_del
>>>> +
>>>> + Description : This routine free element allocated in list of filter
>>>> +
>>>> + Arguments : pointer to the list of filters records anchor.
>>>> +
>>>> + Return Values : None
>>>> +
>>>> + Notes :
>>>> +******************************************************************************/
>>>>
>>>> +static void ntfa_filter_list_del(ntfa_filter_hdl_rec_t **plstr_hdl)
>>>> +{
>>>> + ntfa_filter_hdl_rec_t *lstr_hdl = *plstr_hdl;
>>>> +
>>>> + while (lstr_hdl != NULL) {
>>>> + ntfa_filter_destructor(lstr_hdl);
>>>> + lstr_hdl = lstr_hdl->next;
>>>> + }
>>>> +}
>>>> +/****************************************************************************
>>>>
>>>> + Name : ntfa_filter_hdl_rec_list_del
>>>> +
>>>> + Description : This routine deletes a list of allocated filters
>>>> +
>>>> + Arguments : pointer to the list of filters records anchor.
>>>> +
>>>> + Return Values : None
>>>> +
>>>> + Notes :
>>>> +******************************************************************************/
>>>>
>>>> +static void ntfa_filter_hdl_rec_list_del(ntfa_filter_hdl_rec_t
>>>> **plstr_hdl)
>>>> +{
>>>> + ntfa_filter_hdl_rec_t *lstr_hdl;
>>>> +
>>>> + while ((lstr_hdl = *plstr_hdl) != NULL) {
>>>> + *plstr_hdl = lstr_hdl->next;
>>>> + ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, lstr_hdl->filter_hdl);
>>>> + free(lstr_hdl);
>>>> + lstr_hdl = NULL;
>>>> + }
>>>> +}
>>>> +/****************************************************************************
>>>>
>>>> + Name : ntfa_reader_hdl_rec_list_del
>>>> +
>>>> + Description : This routine deletes a list of allocated readers
>>>> +
>>>> + Arguments : pointer to the list of readers records anchor.
>>>> +
>>>> + Return Values : None
>>>> +
>>>> + Notes :
>>>> +******************************************************************************/
>>>>
>>>> +static void ntfa_reader_hdl_rec_list_del(ntfa_reader_hdl_rec_t
>>>> **plstr_hdl)
>>>> +{
>>>> + ntfa_reader_hdl_rec_t *lstr_hdl;
>>>> + while ((lstr_hdl = *plstr_hdl) != NULL) {
>>>> + *plstr_hdl = lstr_hdl->next;
>>>> + ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, lstr_hdl->reader_hdl);
>>>> + ntfa_del_ntf_filter_ptrs(&lstr_hdl->filters);
>>>> + free(lstr_hdl);
>>>> + lstr_hdl = NULL;
>>>> + }
>>>> +}
>>>>
>>>> static SaAisErrorT
>>>> ntfa_alloc_callback_notification(SaNtfNotificationsT *notification,
>>>> ntfsv_send_not_req_t *not_cbk,
>>>> ntfa_client_hdl_rec_t *hdl_rec)
>>>> @@ -407,6 +489,10 @@ static SaAisErrorT ntfa_hdl_cbk_rec_prc(
>>>> free(cbk_info->param.discarded_cbk.discardedNotificationIdentifiers);
>>>> }
>>>> break;
>>>> + case NTFSV_DUMMY_CALLBACK:
>>>> + TRACE("Do nothing with dummy callback, just return OK");
>>>> + rc = SA_AIS_OK;
>>>> + break;
>>>> default:
>>>> TRACE("unsupported callback type: %d", cbk_info->type);
>>>> rc = SA_AIS_ERR_LIBRARY;
>>>> @@ -544,20 +630,21 @@ unsigned int ntfa_startup(void)
>>>> *
>>>> * @return unsigned int
>>>> */
>>>> -unsigned int ntfa_shutdown(void)
>>>> +unsigned int ntfa_shutdown(bool forced)
>>>> {
>>>> unsigned int rc = NCSCC_RC_SUCCESS;
>>>>
>>>> - TRACE_ENTER2("ntfa_use_count: %u", ntfa_use_count);
>>>> + TRACE_ENTER2("ntfa_use_count: %u, forced: %u", ntfa_use_count,
>>>> forced);
>>>> pthread_mutex_lock(&ntfa_lock);
>>>>
>>>> - if (ntfa_use_count > 1) {
>>>> - /* Users still exist, just decrement the use count */
>>>> - ntfa_use_count--;
>>>> - } else if (ntfa_use_count == 1) {
>>>> + if ((forced && (ntfa_use_count > 0)) || (ntfa_use_count == 1)) {
>>>> ntfa_destroy();
>>>> rc = ncs_agents_shutdown();
>>>> ntfa_use_count = 0;
>>>> + ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_NONE;
>>>> + } else if (ntfa_use_count > 1) {
>>>> + /* Users still exist, just decrement the use count */
>>>> + ntfa_use_count--;
>>>> }
>>>>
>>>> pthread_mutex_unlock(&ntfa_lock);
>>>> @@ -633,7 +720,27 @@ ntfa_client_hdl_rec_t *ntfa_find_hdl_rec
>>>>
>>>> return NULL;
>>>> }
>>>> +/****************************************************************************
>>>>
>>>> + Name : ntfa_subscriber_list_del
>>>>
>>>> + Description : This routine deletes a list of allocated subscribers
>>>> +
>>>> + Arguments : pointer to the list of subscribers records anchor.
>>>> +
>>>> + Return Values : None
>>>> +
>>>> + Notes :
>>>> +******************************************************************************/
>>>>
>>>> +void ntfa_subscriber_list_del()
>>>> +{
>>>> + ntfa_subscriber_list_t *listPtr = subscriberNoList;
>>>> + while (listPtr != NULL) {
>>>> + ntfa_subscriber_list_t* tmpSub = listPtr;
>>>> + listPtr = listPtr->next;
>>>> + free(tmpSub);
>>>> + }
>>>> + subscriberNoList = NULL;
>>>> +}
>>>> /****************************************************************************
>>>>
>>>> Name : ntfa_hdl_list_del
>>>>
>>>> @@ -651,15 +758,24 @@ void ntfa_hdl_list_del(ntfa_client_hdl_r
>>>>
>>>> while ((client_hdl = *p_client_hdl) != NULL) {
>>>> *p_client_hdl = client_hdl->next;
>>>> + m_NCS_IPC_DETACH(&client_hdl->mbx, ntfa_clear_mbx, NULL);
>>>> + m_NCS_IPC_RELEASE(&client_hdl->mbx, NULL);
>>>> ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, client_hdl->local_hdl);
>>>> /** clean up the channel records for this ntfa-client
>>>> **/
>>>> + ntfa_notification_list_del(&client_hdl->notification_list);
>>>> ntfa_notification_hdl_rec_list_del(&client_hdl->notification_list);
>>>> +
>>>> + ntfa_filter_list_del(&client_hdl->filter_list);
>>>> + ntfa_filter_hdl_rec_list_del(&client_hdl->filter_list);
>>>> +
>>>> + ntfa_reader_hdl_rec_list_del(&client_hdl->reader_list);
>>>> /** remove the association with hdl-mngr
>>>> **/
>>>> free(client_hdl);
>>>> client_hdl = 0;
>>>> }
>>>> + ntfa_subscriber_list_del();
>>>> }
>>>>
>>>> /****************************************************************************
>>>>
>>>> @@ -757,10 +873,80 @@ uint32_t ntfa_filter_hdl_rec_del(ntfa_fi
>>>> TRACE("The node couldn't be deleted");
>>>> return NCSCC_RC_FAILURE;
>>>> }
>>>> +/****************************************************************************
>>>>
>>>> + Name : ntfa_hdl_rec_force_del
>>>>
>>>> + Description : This routine deletes all memory allocated to client.
>>>> +
>>>> + Arguments : NTFA_CLIENT_HDL_REC **list_head
>>>> + NTFA_CLIENT_HDL_REC *rm_node
>>>> +
>>>> + Return Values : None
>>>> +
>>>> + Notes :
>>>> +******************************************************************************/
>>>>
>>>> +void ntfa_hdl_rec_force_del(ntfa_client_hdl_rec_t **list_head,
>>>> ntfa_client_hdl_rec_t *rm_node)
>>>> +{
>>>> + ntfa_client_hdl_rec_t *list_iter = *list_head;
>>>> + TRACE_ENTER();
>>>> + /* First remove the rm_node out of the list of client */
>>>> + if (list_iter == rm_node)
>>>> + *list_head = rm_node->next;
>>>> + else {
>>>> + while (list_iter) {
>>>> + if (list_iter->next == rm_node) {
>>>> + list_iter->next = rm_node->next;
>>>> + break;
>>>> + }
>>>> + list_iter = list_iter->next;
>>>> + }
>>>> + }
>>>> + /* Release all msgs in mailbox */
>>>> + ntfsv_msg_t *cbk_msg;
>>>> + while((cbk_msg =
>>>> (ntfsv_msg_t*)m_NCS_IPC_NON_BLK_RECEIVE(&rm_node->mbx, cbk_msg))
>>>> + != NULL) {
>>>> + ntfa_msg_destroy(cbk_msg);
>>>> + }
>>>> + /* delete subscriber of this client out of the subcriberNoList*/
>>>> + ntfa_subscriber_list_t* subscriber_hdl = subscriberNoList;
>>>> + while (subscriber_hdl != NULL) {
>>>> + ntfa_subscriber_list_t *rm_subscriber = subscriber_hdl;
>>>> + subscriber_hdl = subscriber_hdl->next;
>>>> + if (rm_node->local_hdl ==
>>>> rm_subscriber->subscriberListNtfHandle) {
>>>> + if (rm_subscriber->next != NULL) {
>>>> + rm_subscriber->next->prev = rm_subscriber->prev;
>>>> + }
>>>> +
>>>> + if (rm_subscriber->prev != NULL) {
>>>> + rm_subscriber->prev->next = rm_subscriber->next;
>>>> + } else {
>>>> + if (rm_subscriber->next != NULL)
>>>> + subscriberNoList = rm_subscriber->next;
>>>> + else
>>>> + subscriberNoList = NULL;
>>>> + }
>>>> + ntfa_del_ntf_filter_ptrs(&rm_subscriber->filters);
>>>> + free(rm_subscriber);
>>>> + }
>>>> + }
>>>> + /* Now delete client */
>>>> + m_NCS_IPC_DETACH(&rm_node->mbx, ntfa_clear_mbx, NULL);
>>>> + m_NCS_IPC_RELEASE(&rm_node->mbx, NULL);
>>>> + ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node->local_hdl);
>>>> + ntfa_notification_list_del(&rm_node->notification_list);
>>>> + ntfa_notification_hdl_rec_list_del(&rm_node->notification_list);
>>>> +
>>>> + ntfa_filter_list_del(&rm_node->filter_list);
>>>> + ntfa_filter_hdl_rec_list_del(&rm_node->filter_list);
>>>> +
>>>> + ntfa_reader_hdl_rec_list_del(&rm_node->reader_list);
>>>> + free(rm_node);
>>>> +
>>>> + TRACE_LEAVE();
>>>> +}
>>>> /****************************************************************************
>>>>
>>>> Name : ntfa_hdl_rec_del
>>>> -
>>>> +
>>>> Description : This routine deletes the a client handle record from
>>>> a list of client hdl records.
>>>>
>>>> @@ -962,7 +1148,7 @@ ntfa_client_hdl_rec_t *ntfa_hdl_rec_add(
>>>> /** Associate with the client_id obtained from NTFS
>>>> **/
>>>> rec->ntfs_client_id = client_id;
>>>> -
>>>> + rec->valid = true;
>>>> /** Initialize and attach the IPC/Priority queue
>>>> **/
>>>>
>>>> @@ -1063,7 +1249,7 @@ static void logtrace_init_constructor(vo
>>>> *
>>>> * @param instance
>>>> */
>>>> -void ntfa_hdl_rec_destructor(ntfa_notification_hdl_rec_t *instance)
>>>> +void ntfa_notification_destructor(ntfa_notification_hdl_rec_t
>>>> *instance)
>>>> {
>>>> ntfa_notification_hdl_rec_t *notificationInstance = instance;
>>>>
>>>> @@ -1111,7 +1297,7 @@ void ntfa_hdl_rec_destructor(ntfa_notifi
>>>> *
>>>> * @param instance
>>>> */
>>>> -void ntfa_filter_hdl_rec_destructor(ntfa_filter_hdl_rec_t *filter_rec)
>>>> +void ntfa_filter_destructor(ntfa_filter_hdl_rec_t *filter_rec)
>>>> {
>>>> switch (filter_rec->ntfType) {
>>>> case SA_NTF_TYPE_OBJECT_CREATE_DELETE:
>>>> @@ -1202,6 +1388,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>>>> **/
>>>> ncshm_give_hdl(rm_node->reader_hdl);
>>>> ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA, rm_node->reader_hdl);
>>>> + ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>>>> free(rm_node);
>>>> return NCSCC_RC_SUCCESS;
>>>> } else { /* find the rec */
>>>> @@ -1213,6 +1400,7 @@ uint32_t ntfa_reader_hdl_rec_del(ntfa_re
>>>> **/
>>>> ncshm_give_hdl(rm_node->reader_hdl);
>>>> ncshm_destroy_hdl(NCS_SERVICE_ID_NTFA,
>>>> rm_node->reader_hdl);
>>>> + ntfa_del_ntf_filter_ptrs(&rm_node->filters);
>>>> free(rm_node);
>>>> return NCSCC_RC_SUCCESS;
>>>> }
>>>> @@ -1258,3 +1446,258 @@ void ntfa_add_to_async_cbk_msg_list(ntfs
>>>>
>>>> TRACE_LEAVE();
>>>> }
>>>> +
>>>> +/****************************************************************************
>>>>
>>>> + Name : ntfa_notify_handle_invalid
>>>> +
>>>> + Description : This routine sends a dummy callback msg to
>>>> client's mailbox
>>>> + so that the client polls in and calls saNtfDispatch.
>>>> + Arguments :
>>>> +
>>>> + Return Values : None
>>>> +
>>>> + Notes : None
>>>> +******************************************************************************/
>>>>
>>>> +void ntfa_notify_handle_invalid() {
>>>> + ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
>>>> + TRACE_ENTER();
>>>> + while (client_hdl != NULL) {
>>>> + /* Only applicable for subscriber */
>>>> + if (client_hdl->reg_cbk.saNtfNotificationCallback != NULL ||
>>>> + client_hdl->reg_cbk.saNtfNotificationDiscardedCallback != NULL) {
>>>> + /* Create a dummy msg */
>>>> + ntfsv_msg_t *msg = malloc(sizeof(ntfsv_msg_t));
>>>> + memset(msg, 0, sizeof(ntfsv_msg_t));
>>>> + msg->info.cbk_info.type = NTFSV_DUMMY_CALLBACK;
>>>> + /* Send dummy msg to client mailbox */
>>> [Praveen] Is this dummy callback being sent to client in a way to
>>> force it to recover client and subscriber?
>> [Minh] Because agent doesn't recover client in mds callback, so in case
>> of subscriber who is just waiting for notification, agent needs to send
>> dummy callback to client's mailbox to wake up client's polling loop.
>> Then client will call saAmfDispatch, and from there agent will recover
>> client handles. This dummy callback is invisible to client.
>>>> + if (m_NCS_IPC_SEND(&client_hdl->mbx, msg,
>>>> MDS_SEND_PRIORITY_HIGH) != NCSCC_RC_SUCCESS) {
>>>> + TRACE_1("m_NCS_IPC_SEND Failed to client(id:%u)",
>>>> client_hdl->ntfs_client_id);
>>>> + ntfa_msg_destroy(msg);
>>>> + }
>>>> + }
>>>> +
>>>> + client_hdl = client_hdl->next;
>>>> + }
>>>> + TRACE_LEAVE();
>>>> +}
>>>> +/****************************************************************************
>>>>
>>>> + Name : ntfa_update_ntfsv_state
>>>> +
>>>> + Description : Update current NTF Server state by the
>>>> @changedState indicated
>>>> + by MDS event
>>>> + Arguments : @changedState [IN]: state to be changed of NTF Server
>>>> +
>>>> + Return Values : None
>>>> +
>>>> + Notes : None
>>>> +******************************************************************************/
>>>>
>>>> +void ntfa_update_ntfsv_state(ntfa_ntfsv_state_t changedState)
>>>> +{
>>>> + TRACE_ENTER();
>>>> + TRACE_1("Current state: %u, Changed state: %u",
>>>> ntfa_cb.ntfa_ntfsv_state,
>>>> + changedState);
>>>> +
>>>> + ntfa_client_hdl_rec_t *client_hdl = ntfa_cb.client_list;
>>>> +
>>>> + switch (ntfa_cb.ntfa_ntfsv_state){
>>>> + case NTFA_NTFSV_NONE:
>>>> + ntfa_cb.ntfa_ntfsv_state = changedState;
>>>> + break;
>>>> + case NTFA_NTFSV_DOWN:
>>>> + if (changedState == NTFA_NTFSV_NEW_ACTIVE ||
>>>> + changedState == NTFA_NTFSV_UP) {
>>>> + TRACE("Active NTF server has been restarted");
>>>> + ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_UP;
>>>> + ntfa_notify_handle_invalid();
>>>> + } else
>>>> + TRACE("Unexpected state changes");
>>>> + break;
>>>> + case NTFA_NTFSV_NO_ACTIVE:
>>>> + if (changedState == NTFA_NTFSV_NEW_ACTIVE) {
>>>> + TRACE("Standby NTF server becomes new Active");
>>>> + /* NTF server is functioning normally */
>>>> + ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_UP;
>>>> + } else if (changedState == NTFA_NTFSV_DOWN) {
>>>> + TRACE("Active NTF Server is Down");
>>>> + ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_DOWN;
>>>> + /* Mark all client handles are invalid */
>>>> + while (client_hdl != NULL) {
>>>> + client_hdl->valid = false;
>>>> + client_hdl = client_hdl->next;
>>>> + }
>>>> + }
>>>> + break;
>>>> + case NTFA_NTFSV_NEW_ACTIVE:
>>>> + TRACE("Unknown");
>>>> + break;
>>>> + case NTFA_NTFSV_UP:
>>>> + if (changedState == NTFA_NTFSV_NO_ACTIVE) {
>>>> + TRACE("Active NTF server temporarily unavailable");
>>>> + /* Failover/Switchover is happening
>>>> + * Any API calls result in TRY_AGAIN
>>>> + */
>>>> + ntfa_cb.ntfa_ntfsv_state = NTFA_NTFSV_NO_ACTIVE;
>>>> + } else
>>>> + TRACE("Unexpected state changes");
>>>> + break;
>>>> + default:
>>>> + osafassert(false);
>>>> + }
>>>> +
>>>> + TRACE_LEAVE();
>>>> +}
>>>> +/****************************************************************************
>>>>
>>>> + Name : ntfa_copy_ntf_filter_ptrs
>>>> +
>>>> + Description : Copy a list of filter from pSrc to pDes
>>>> +
>>>> + Arguments : pDes* [OUT]: list of outcome filter
>>>> + pSrc* [IN]: list of input filter
>>>> +
>>>> + Return Values : SA_AIS_OK if succeed, other values as failed
>>>> +
>>>> + Notes : None
>>>> +******************************************************************************/
>>>>
>>>> +SaAisErrorT ntfa_copy_ntf_filter_ptrs(ntfsv_filter_ptrs_t* pDes,
>>>> + const ntfsv_filter_ptrs_t* pSrc) {
>>>> + SaAisErrorT rc = SA_AIS_OK;
>>>> + SaNtfNotificationFilterHeaderT *des_header;
>>>> + SaNtfNotificationFilterHeaderT *src_header;
>>>> + TRACE_ENTER();
>>>> +
>>>> + if (pSrc->alarm_filter) {
>>>> + pDes->alarm_filter = calloc(1,
>>>> sizeof(SaNtfAlarmNotificationFilterT));
>>>> + des_header = &(pDes->alarm_filter->notificationFilterHeader);
>>>> + src_header = &(pSrc->alarm_filter->notificationFilterHeader);
>>>> + if ((rc = ntfsv_filter_header_alloc(des_header,
>>>> src_header->numEventTypes,
>>>> + src_header->numNotificationObjects,
>>>> + src_header->numNotifyingObjects,
>>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>>> + goto done;
>>>> + if ((rc = ntfsv_filter_alarm_alloc(pDes->alarm_filter,
>>>> + pSrc->alarm_filter->numProbableCauses,
>>>> + pSrc->alarm_filter->numPerceivedSeverities,
>>>> + pSrc->alarm_filter->numTrends)) != SA_AIS_OK)
>>>> + goto done;
>>>> + if ((rc = ntfsv_copy_ntf_filter_alarm(pDes->alarm_filter,
>>>> + pSrc->alarm_filter)) != SA_AIS_OK)
>>>> + goto done;
>>>> + }
>>>> +
>>>> + if (pSrc->sec_al_filter) {
>>>> + pDes->sec_al_filter = calloc(1,
>>>> sizeof(SaNtfSecurityAlarmNotificationFilterT));
>>>> + des_header = &(pDes->sec_al_filter->notificationFilterHeader);
>>>> + src_header = &(pSrc->sec_al_filter->notificationFilterHeader);
>>>> + if ((rc = ntfsv_filter_header_alloc(des_header,
>>>> src_header->numEventTypes,
>>>> + src_header->numNotificationObjects,
>>>> + src_header->numNotifyingObjects,
>>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>>> + goto done;
>>>> + if ((rc = ntfsv_filter_sec_alarm_alloc(pDes->sec_al_filter,
>>>> + pSrc->sec_al_filter->numProbableCauses,
>>>> + pSrc->sec_al_filter->numSeverities,
>>>> + pSrc->sec_al_filter->numSecurityAlarmDetectors,
>>>> + pSrc->sec_al_filter->numServiceUsers,
>>>> + pSrc->sec_al_filter->numServiceProviders)) != SA_AIS_OK)
>>>> + goto done;
>>>> + if ((rc = ntfsv_copy_ntf_filter_sec_alarm(pDes->sec_al_filter,
>>>> + pSrc->sec_al_filter)) != SA_AIS_OK)
>>>> + goto done;
>>>> + }
>>>> +
>>>> + if (pSrc->sta_ch_filter) {
>>>> + pDes->sta_ch_filter = calloc(1,
>>>> sizeof(SaNtfStateChangeNotificationFilterT));
>>>> + des_header = &(pDes->sta_ch_filter->notificationFilterHeader);
>>>> + src_header = &(pSrc->sta_ch_filter->notificationFilterHeader);
>>>> + if ((rc = ntfsv_filter_header_alloc(des_header,
>>>> src_header->numEventTypes,
>>>> + src_header->numNotificationObjects,
>>>> + src_header->numNotifyingObjects,
>>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>>> + goto done;
>>>> + if ((rc = ntfsv_filter_state_ch_alloc(pDes->sta_ch_filter,
>>>> + pSrc->sta_ch_filter->numSourceIndicators,
>>>> + pSrc->sta_ch_filter->numStateChanges)) != SA_AIS_OK)
>>>> + goto done;
>>>> + if ((rc = ntfsv_copy_ntf_filter_state_ch(pDes->sta_ch_filter,
>>>> + pSrc->sta_ch_filter)) != SA_AIS_OK)
>>>> + goto done;
>>>> + }
>>>> +
>>>> + if (pSrc->obj_cr_del_filter) {
>>>> + pDes->obj_cr_del_filter = calloc(1,
>>>> sizeof(SaNtfObjectCreateDeleteNotificationFilterT));
>>>> + des_header =
>>>> &(pDes->obj_cr_del_filter->notificationFilterHeader);
>>>> + src_header =
>>>> &(pSrc->obj_cr_del_filter->notificationFilterHeader);
>>>> + if ((rc = ntfsv_filter_header_alloc(des_header,
>>>> src_header->numEventTypes,
>>>> + src_header->numNotificationObjects,
>>>> + src_header->numNotifyingObjects,
>>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>>> + goto done;
>>>> + if ((rc =
>>>> ntfsv_filter_obj_cr_del_alloc(pDes->obj_cr_del_filter,
>>>> + pSrc->obj_cr_del_filter->numSourceIndicators)) != SA_AIS_OK)
>>>> + goto done;
>>>> + if ((rc =
>>>> ntfsv_copy_ntf_filter_obj_cr_del(pDes->obj_cr_del_filter,
>>>> + pSrc->obj_cr_del_filter)) != SA_AIS_OK)
>>>> + goto done;
>>>> + }
>>>> +
>>>> + if (pSrc->att_ch_filter) {
>>>> + pDes->att_ch_filter = calloc(1,
>>>> sizeof(SaNtfAttributeChangeNotificationFilterT));
>>>> + des_header = &(pDes->att_ch_filter->notificationFilterHeader);
>>>> + src_header = &(pSrc->att_ch_filter->notificationFilterHeader);
>>>> + if ((rc = ntfsv_filter_header_alloc(des_header,
>>>> src_header->numEventTypes,
>>>> + src_header->numNotificationObjects,
>>>> + src_header->numNotifyingObjects,
>>>> + src_header->numNotificationClassIds)) != SA_AIS_OK)
>>>> + goto done;
>>>> + if ((rc = ntfsv_filter_attr_change_alloc(pDes->att_ch_filter,
>>>> + pSrc->att_ch_filter->numSourceIndicators)) != SA_AIS_OK)
>>>> + goto done;
>>>> + if ((rc = ntfsv_copy_ntf_filter_attr_ch(pDes->att_ch_filter,
>>>> + pSrc->att_ch_filter)) != SA_AIS_OK)
>>>> + goto done;
>>>> + }
>>>> +done:
>>>> + TRACE_LEAVE();
>>>> + return rc;
>>>> +}
>>>> +
>>>> +/****************************************************************************
>>>>
>>>> + Name : ntfa_del_ntf_filter_ptrs
>>>> +
>>>> + Description : Delete the filter pointers
>>>> +
>>>> + Arguments : filter_ptrs* [IN/OUT]: filter pointers
>>>> +
>>>> + Return Values : SA_AIS_OK if succeed, other values as failed
>>>> +
>>>> + Notes : None
>>>> +******************************************************************************/
>>>>
>>>> +SaAisErrorT ntfa_del_ntf_filter_ptrs(ntfsv_filter_ptrs_t* filter_ptrs)
>>>> +{
>>>> + SaAisErrorT rc = SA_AIS_OK;
>>>> + if (filter_ptrs->alarm_filter) {
>>>> + ntfsv_filter_alarm_free(filter_ptrs->alarm_filter, true);
>>>> + free(filter_ptrs->alarm_filter);
>>>> + }
>>>> +
>>>> + if (filter_ptrs->sec_al_filter) {
>>>> + ntfsv_filter_sec_alarm_free(filter_ptrs->sec_al_filter, true);
>>>> + free(filter_ptrs->sec_al_filter);
>>>> + }
>>>> +
>>>> + if (filter_ptrs->sta_ch_filter) {
>>>> + ntfsv_filter_state_ch_free(filter_ptrs->sta_ch_filter, true);
>>>> + free(filter_ptrs->sta_ch_filter);
>>>> + }
>>>> +
>>>> + if (filter_ptrs->obj_cr_del_filter) {
>>>> + ntfsv_filter_obj_cr_del_free(filter_ptrs->obj_cr_del_filter, true);
>>>> + free(filter_ptrs->obj_cr_del_filter);
>>>> + }
>>>> +
>>>> + if (filter_ptrs->att_ch_filter) {
>>>> + ntfsv_filter_attr_ch_free(filter_ptrs->att_ch_filter, true);
>>>> + free(filter_ptrs->att_ch_filter);
>>>> + }
>>>> + return rc;
>>>> +}
>>>>
>>>
>>
>
> ------------------------------------------------------------------------------
> Site24x7 APM Insight: Get Deep Visibility into Application Performance
> APM + Mobile APM + RUM: Monitor 3 App instances at just $35/Month
> Monitor end-to-end web transactions and take corrective actions now
> Troubleshoot faster and improve end-user experience. Signup Now!
> http://pubads.g.doubleclick.net/gampad/clk?id=272487151&iu=/4140
> _______________________________________________
> Opensaf-devel mailing list
> Opensaf-devel@lists.sourceforge.net
> https://lists.sourceforge.net/lists/listinfo/opensaf-devel
>
------------------------------------------------------------------------------
Site24x7 APM Insight: Get Deep Visibility into Application Performance
APM + Mobile APM + RUM: Monitor 3 App instances at just $35/Month
Monitor end-to-end web transactions and take corrective actions now
Troubleshoot faster and improve end-user experience. Signup Now!
http://pubads.g.doubleclick.net/gampad/clk?id=272487151&iu=/4140
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel
