Hi Canh, I will do a small change based on the comment before pushing. Also see my comment [Lennart]
Thanks Lennart From: Canh Van Truong [mailto:canh.v.tru...@dektech.com.au] Sent: den 6 april 2018 08:29 To: Lennart Lund <lennart.l...@ericsson.com>; Vu Minh Nguyen <vu.m.ngu...@dektech.com.au> Cc: opensaf-devel@lists.sourceforge.net Subject: RE: [PATCH 1/1] log: Handling of IMM OI BAD HANDLE in log server is incorrect [#2799] Hi Lennart, Ack with one comment [Canh] Thanks Canh -----Original Message----- From: Lennart Lund [mailto:lennart.l...@ericsson.com] Sent: Thursday, April 5, 2018 7:38 PM To: vu.m.ngu...@dektech.com.au<mailto:vu.m.ngu...@dektech.com.au>; canh.v.tru...@dektech.com.au<mailto:canh.v.tru...@dektech.com.au> Cc: opensaf-devel@lists.sourceforge.net<mailto:opensaf-devel@lists.sourceforge.net>; Lennart Lund <lennart.l...@ericsson.com<mailto:lennart.l...@ericsson.com>> Subject: [PATCH 1/1] log: Handling of IMM OI BAD HANDLE in log server is incorrect [#2799] Recovery of OI handle shall be started in all places where BAD HANDLE can be returned. Creation of OI must be done in background thread. Ongoing creation must be possible to stop e.g if server is becoming standby --- src/log/Makefile.am | 3 + src/log/logd/lgs.h | 24 --- src/log/logd/lgs_amf.cc | 25 +-- src/log/logd/lgs_cb.h | 3 - src/log/logd/lgs_config.cc | 28 ++- src/log/logd/lgs_config.h | 5 +- src/log/logd/lgs_evt.cc | 56 +++--- src/log/logd/lgs_imm.cc | 333 ++++--------------------------- src/log/logd/lgs_imm.h | 53 +++++ src/log/logd/lgs_main.cc | 116 ++++------- src/log/logd/lgs_mbcsv_v2.cc | 7 +- src/log/logd/lgs_mbcsv_v3.cc | 7 +- src/log/logd/lgs_mbcsv_v5.cc | 3 +- src/log/logd/lgs_oi_admin.cc | 464 +++++++++++++++++++++++++++++++++++++++++++ src/log/logd/lgs_oi_admin.h | 108 ++++++++++ src/log/logd/lgs_recov.cc | 4 +- src/log/logd/lgs_stream.cc | 85 ++++++-- 17 files changed, 857 insertions(+), 467 deletions(-) create mode 100644 src/log/logd/lgs_imm.h create mode 100644 src/log/logd/lgs_oi_admin.cc create mode 100644 src/log/logd/lgs_oi_admin.h diff --git a/src/log/Makefile.am b/src/log/Makefile.am index 3d951eb5d..5d33d355b 100644 --- a/src/log/Makefile.am +++ b/src/log/Makefile.am @@ -79,6 +79,7 @@ noinst_HEADERS += \ src/log/logd/lgs_file.h \ src/log/logd/lgs_filehdl.h \ src/log/logd/lgs_fmt.h \ + src/log/logd/lgs_imm.h \ src/log/logd/lgs_imm_gcfg.h \ src/log/logd/lgs_mbcsv.h \ src/log/logd/lgs_mbcsv_v1.h \ @@ -86,6 +87,7 @@ noinst_HEADERS += \ src/log/logd/lgs_mbcsv_v3.h \ src/log/logd/lgs_mbcsv_v5.h \ src/log/logd/lgs_mbcsv_v6.h \ + src/log/logd/lgs_oi_admin.h \ src/log/logd/lgs_recov.h \ src/log/logd/lgs_stream.h \ src/log/logd/lgs_util.h \ @@ -139,6 +141,7 @@ bin_osaflogd_SOURCES = \ src/log/logd/lgs_mbcsv_v5.cc \ src/log/logd/lgs_mbcsv_v6.cc \ src/log/logd/lgs_mds.cc \ + src/log/logd/lgs_oi_admin.cc \ src/log/logd/lgs_recov.cc \ src/log/logd/lgs_stream.cc \ src/log/logd/lgs_util.cc \ diff --git a/src/log/logd/lgs.h b/src/log/logd/lgs.h index 18e6d9281..b1d773375 100644 --- a/src/log/logd/lgs.h +++ b/src/log/logd/lgs.h @@ -95,7 +95,6 @@ extern uint32_t mbox_msgs[NCS_IPC_PRIORITY_MAX]; extern bool mbox_full[NCS_IPC_PRIORITY_MAX]; extern uint32_t mbox_low[NCS_IPC_PRIORITY_MAX]; extern pthread_mutex_t lgs_mbox_init_mutex; -extern pthread_mutex_t lgs_OI_init_mutex; extern uint32_t initialize_for_assignment(lgs_cb_t *cb, SaAmfHAStateT ha_state); @@ -108,27 +107,4 @@ extern uint32_t lgs_mds_msg_send(lgs_cb_t *cb, lgsv_msg_t *msg, MDS_DEST *dest, MDS_SYNC_SND_CTXT *mds_ctxt, MDS_SEND_PRIORITY_TYPE prio); -extern SaAisErrorT lgs_imm_create_configStream(lgs_cb_t *cb); -extern void logRootDirectory_filemove(const std::string &new_logRootDirectory, - const std::string &old_logRootDirectory, - time_t *cur_time_in); -extern void logDataGroupname_fileown(const char *new_logDataGroupname); - -extern void lgs_imm_impl_reinit_nonblocking(lgs_cb_t *cb); -extern void lgs_imm_init_OI_handle(SaImmOiHandleT *immOiHandle, - SaSelectionObjectT *immSelectionObject); -extern void lgs_imm_impl_set(SaImmOiHandleT *immOiHandle, - SaSelectionObjectT *immSelectionObject); -extern SaAisErrorT lgs_imm_init_configStreams(lgs_cb_t *cb); - -// Functions for recovery handling -void lgs_cleanup_abandoned_streams(); -void lgs_delete_one_stream_object(const std::string &name_str); -void lgs_search_stream_objects(); -SaUint32T *lgs_get_scAbsenceAllowed_attr(SaUint32T *attr_val); -int lgs_get_streamobj_attr(SaImmAttrValuesT_2 ***attrib_out, - const std::string &object_name, - SaImmHandleT *immOmHandle); -int lgs_free_streamobj_attr(SaImmHandleT immHandle); - #endif // LOG_LOGD_LGS_H_ diff --git a/src/log/logd/lgs_amf.cc b/src/log/logd/lgs_amf.cc index 6fa044ff2..999989146 100644 --- a/src/log/logd/lgs_amf.cc +++ b/src/log/logd/lgs_amf.cc @@ -23,6 +23,7 @@ #include "osaf/immutil/immutil.h" #include "log/logd/lgs.h" #include "log/logd/lgs_config.h" +#include "log/logd/lgs_oi_admin.h" static void close_all_files() { log_stream_t *stream; @@ -63,8 +64,7 @@ static SaAisErrorT amf_active_state_handler(lgs_cb_t *cb, goto done; } - lgs_imm_impl_set(&cb->immOiHandle, &cb->immSelectionObject); - conf_runtime_obj_create(cb->immOiHandle); + lgsOiStart(); lgs_start_gcfg_applier(); // Iterate all existing log streams in cluster. @@ -127,12 +127,8 @@ static SaAisErrorT amf_quiescing_state_handler(lgs_cb_t *cb, TRACE_ENTER2("HA QUIESCING request"); close_all_files(); - /* Give up our IMM OI implementer role */ - SaAisErrorT ais_rc = immutil_saImmOiImplementerClear(cb->immOiHandle); - if (ais_rc != SA_AIS_OK) { - LOG_WA("immutil_saImmOiImplementerClear failed: %s", saf_error(ais_rc)); - } - + // Give up our IMM OI implementer role and the OpensafConfig class applier + lgsOiStop(); lgs_stop_gcfg_applier(); return saAmfCSIQuiescingComplete(cb->amf_hdl, invocation, SA_AIS_OK); @@ -158,11 +154,7 @@ static SaAisErrorT amf_quiesced_state_handler(lgs_cb_t *cb, close_all_files(); /* Give up our IMM OI implementer role */ - SaAisErrorT rc = immutil_saImmOiImplementerClear(cb->immOiHandle); - if (rc != SA_AIS_OK) { - LOG_WA("immutil_saImmOiImplementerClear failed: %s", saf_error(rc)); - } - + lgsOiStop(); lgs_stop_gcfg_applier(); /* @@ -173,9 +165,10 @@ static SaAisErrorT amf_quiesced_state_handler(lgs_cb_t *cb, mds_role = cb->mds_role; cb->mds_role = V_DEST_RL_QUIESCED; + SaAisErrorT ais_rc = SA_AIS_OK; if (lgs_mds_change_role(cb) != NCSCC_RC_SUCCESS) { LOG_ER("lgs_mds_change_role FAILED"); - rc = SA_AIS_ERR_FAILED_OPERATION; + ais_rc = SA_AIS_ERR_FAILED_OPERATION; cb->mds_role = mds_role; goto done; } @@ -183,8 +176,9 @@ static SaAisErrorT amf_quiesced_state_handler(lgs_cb_t *cb, cb->amf_invocation_id = invocation; cb->is_quiesced_set = true; done: - return rc; + return ais_rc; } + /**************************************************************************** * Name : amf_health_chk_callback * @@ -207,6 +201,7 @@ static void amf_health_chk_callback(SaInvocationT invocation, SaAmfHealthcheckKeyT *checkType) { saAmfResponse(lgs_cb->amf_hdl, invocation, SA_AIS_OK); } + /**************************************************************************** * Name : amf_csi_set_callback * diff --git a/src/log/logd/lgs_cb.h b/src/log/logd/lgs_cb.h index f2602c2d8..1b1a67d9c 100644 --- a/src/log/logd/lgs_cb.h +++ b/src/log/logd/lgs_cb.h @@ -81,9 +81,6 @@ typedef struct lgs_cb { SaInvocationT amf_invocation_id; /* AMF InvocationID - needed to handle Quiesed state */ bool is_quiesced_set; - SaImmOiHandleT immOiHandle; /* IMM OI handle */ - SaSelectionObjectT - immSelectionObject; /* Selection Object to wait for IMM events */ SaSelectionObjectT clmSelectionObject; /* Selection Object to wait for clms events */ SaClmHandleT clm_hdl; /* CLM handle, obtained through CLM init */ diff --git a/src/log/logd/lgs_config.cc b/src/log/logd/lgs_config.cc index 4190e3048..8d9119e2f 100644 --- a/src/log/logd/lgs_config.cc +++ b/src/log/logd/lgs_config.cc @@ -41,6 +41,7 @@ #include "log/logd/lgs_file.h" #include "log/logd/lgs.h" #include "log/logd/lgs_common.h" +#include "log/logd/lgs_oi_admin.h" /* Mutex for making read and write of configuration data thread safe */ @@ -1338,8 +1339,8 @@ static void read_log_config_environ_var_2() { * Public functions for handling configuration information ******************************************************************************/ -void lgs_cfg_init(SaImmOiHandleT immOiHandle, SaAmfHAStateT ha_state) { - TRACE_ENTER2("immOiHandle = %lld", immOiHandle); +void lgs_cfg_init() { + TRACE_ENTER(); /* Read configuration step 1 * Read all values from the log service configuration object @@ -1585,6 +1586,10 @@ void conf_runtime_obj_create(SaImmOiHandleT immOiHandle) { if (rc == SA_AIS_ERR_EXIST) { TRACE("Server runtime configuration object already exist"); + } else if (rc == SA_AIS_ERR_BAD_HANDLE) { + LOG_NO("%s: saImmOiRtObjectCreate_2 BAD_HANDLE, start re-create", + __FUNCTION__); + lgsOiCreateBackground(); } else if (rc != SA_AIS_OK) { LOG_NO("%s: Cannot create config runtime object %s", __FUNCTION__, saf_error(rc)); @@ -1667,6 +1672,14 @@ static SaAisErrorT update_runtime_attrValues( attrMod.modAttr.attrValueType = attrValueType; attrMod.modAttr.attrValues = values_array; ais_rc = immutil_saImmOiRtObjectUpdate_2(immOiHandle, &objectName, attrMods); + if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + LOG_NO("%s: saImmOiRtObjectUpdate_2() Fail, %s", __FUNCTION__, + saf_error(ais_rc)); // [Canh] The recovery OI here is no meaning because the abort while ais_rc != SA_AIS_OK in function conf_runtime_obj_handler() at line 1817 in lgs_config.cc file (no recovery OI while updating pure runtime attributes) [Lennart] Your comment is correct. To follow the pattern used for immutil_update_one_rattr() an abort should be done here since immutil_saImmOiRtObjectUpdate_2() is updating a cached runtime attribute like immutil_update_one_rattr() but is used if the attribute is multi-value but if nothing is done an abort will be done anyway as described in your comment. I think the most correct to do in this case is to remove the special handling in case of BAD HANDLE and just return the error code as before. + lgsOiCreateBackground(); + } else if (ais_rc != SA_AIS_OK) { + LOG_NO("%s: saImmOiRtObjectUpdate_2() Fail, %s", __FUNCTION__, + saf_error(ais_rc)); + } free(values_array); TRACE_LEAVE(); return ais_rc; @@ -1692,8 +1705,7 @@ static SaAisErrorT update_lgs_cfg_runtime_multivalue( immOiHandle, LGS_CFG_RUNTIME_OBJECT, attributeName, valueType, attrValuesNumber, attrValues); if (ais_rc != SA_AIS_OK) { - LOG_NO("%s: update_runtime_attrValues Fail %s", __FUNCTION__, - saf_error(ais_rc)); + LOG_NO("%s: update_runtime_attrValues Fail", __FUNCTION__); } // Free the memory allocated by vector_of_strings_to_attrValues() @@ -1712,7 +1724,7 @@ static SaAisErrorT update_lgs_cfg_runtime_multivalue( * @param immOiHandle[in] * @param attributeNames[in] */ -void conf_runtime_obj_hdl(SaImmOiHandleT immOiHandle, +void conf_runtime_obj_handler(SaImmOiHandleT immOiHandle, const SaImmAttrNameT *attributeNames) { SaImmAttrNameT attributeName; int i = 0; @@ -1799,9 +1811,9 @@ void conf_runtime_obj_hdl(SaImmOiHandleT immOiHandle, TRACE("%s: unknown attribute %s", __FUNCTION__, attributeName); } - if (ais_rc != SA_AIS_OK) { - LOG_ER("immutil_update_one_rattr (%s) failed: %s", attributeName, - saf_error(ais_rc)); + if (ais_rc != SA_AIS_OK) { + LOG_ER("%s: immutil_update_one_rattr (%s) failed: %s", __FUNCTION__, + attributeName, saf_error(ais_rc)); osaf_abort(0); } } diff --git a/src/log/logd/lgs_config.h b/src/log/logd/lgs_config.h index 6885271a4..3f1b05e51 100644 --- a/src/log/logd/lgs_config.h +++ b/src/log/logd/lgs_config.h @@ -135,7 +135,7 @@ typedef struct config_chkpt { * Read the log service configuration data verify and update configuration * data structure */ -void lgs_cfg_init(SaImmOiHandleT immOiHandle, SaAmfHAStateT ha_state); +void lgs_cfg_init(); /** * Get value of log service configuration parameter from the configuration data @@ -316,8 +316,9 @@ void lgs_groupnameconf_set(const char *data_groupname_str); * Handle runtime object for showing actual configuration and configuration * related information */ + void conf_runtime_obj_create(SaImmOiHandleT immOiHandle); -void conf_runtime_obj_hdl(SaImmOiHandleT immOiHandle, +void conf_runtime_obj_handler(SaImmOiHandleT immOiHandle, const SaImmAttrNameT *attributeNames); /* diff --git a/src/log/logd/lgs_evt.cc b/src/log/logd/lgs_evt.cc index 4b735875d..589cb262b 100644 --- a/src/log/logd/lgs_evt.cc +++ b/src/log/logd/lgs_evt.cc @@ -31,6 +31,7 @@ #include "log/logd/lgs_imm_gcfg.h" #include "log/logd/lgs_clm.h" #include "log/logd/lgs_dest.h" +#include "log/logd/lgs_oi_admin.h" void *client_db = nullptr; /* used for C++ STL map */ @@ -569,7 +570,7 @@ static uint32_t proc_rda_cb_msg(lgsv_lgs_evt_t *evt) { } /* fail over, become implementer */ - lgs_imm_impl_set(&lgs_cb->immOiHandle, &lgs_cb->immSelectionObject); + lgsOiCreateBackground(); lgs_start_gcfg_applier(); /* Agent down list has to be processed first */ @@ -613,7 +614,6 @@ uint32_t lgs_cb_init(lgs_cb_t *lgs_cb) { lgs_cb->fully_initialized = false; lgs_cb->amfSelectionObject = -1; - lgs_cb->immSelectionObject = -1; lgs_cb->mbcsv_sel_obj = -1; lgs_cb->clm_hdl = 0; lgs_cb->clmSelectionObject = -1; @@ -835,7 +835,7 @@ snd_rsp: */ SaAisErrorT create_new_app_stream(lgsv_stream_open_req_t *open_sync_param, log_stream_t **o_stream) { - SaAisErrorT rc = SA_AIS_OK; + SaAisErrorT ais_rc = SA_AIS_OK; log_stream_t *stream; SaBoolT twelveHourModeFlag; SaUint32T logMaxLogrecsize_conf = 0; @@ -848,13 +848,13 @@ SaAisErrorT create_new_app_stream(lgsv_stream_open_req_t *open_sync_param, if (lgs_is_extended_name_valid(&open_sync_param->lstr_name) == false) { TRACE("SaNameT is invalid"); - rc = SA_AIS_ERR_INVALID_PARAM; + ais_rc = SA_AIS_ERR_INVALID_PARAM; goto done; } if (open_sync_param->logFileFullAction != SA_LOG_FILE_FULL_ACTION_ROTATE) { TRACE("Unsupported logFileFullAction"); - rc = SA_AIS_ERR_NOT_SUPPORTED; + ais_rc = SA_AIS_ERR_NOT_SUPPORTED; goto done; } @@ -864,7 +864,7 @@ SaAisErrorT create_new_app_stream(lgsv_stream_open_req_t *open_sync_param, ((open_sync_param->maxFilesRotated < 1) || (open_sync_param->maxFilesRotated > 127))) { TRACE("Invalid maxFilesRotated. Valid Range = [1-127]"); - rc = SA_AIS_ERR_INVALID_PARAM; + ais_rc = SA_AIS_ERR_INVALID_PARAM; goto done; } @@ -880,21 +880,21 @@ SaAisErrorT create_new_app_stream(lgsv_stream_open_req_t *open_sync_param, STREAM_TYPE_APPLICATION_RT, &twelveHourModeFlag)) { TRACE("format expression failure"); - rc = SA_AIS_ERR_INVALID_PARAM; + ais_rc = SA_AIS_ERR_INVALID_PARAM; goto done; } /* Verify if there is any special character in logFileName */ if (lgs_has_special_char(open_sync_param->logFileName) == true) { TRACE("Invalid logFileName - %s", open_sync_param->logFileName); - rc = SA_AIS_ERR_INVALID_PARAM; + ais_rc = SA_AIS_ERR_INVALID_PARAM; goto done; } /* Verify if logFileName length is valid */ if (lgs_is_valid_filelength(open_sync_param->logFileName) == false) { TRACE("logFileName is invalid"); - rc = SA_AIS_ERR_INVALID_PARAM; + ais_rc = SA_AIS_ERR_INVALID_PARAM; goto done; } @@ -902,7 +902,7 @@ SaAisErrorT create_new_app_stream(lgsv_stream_open_req_t *open_sync_param, if (lgs_is_valid_pathlength(open_sync_param->logFilePathName, open_sync_param->logFileName) == false) { TRACE("logFilePathName is invalid"); - rc = SA_AIS_ERR_INVALID_PARAM; + ais_rc = SA_AIS_ERR_INVALID_PARAM; goto done; } @@ -913,7 +913,7 @@ SaAisErrorT create_new_app_stream(lgsv_stream_open_req_t *open_sync_param, if ((stream->fileName == open_sync_param->logFileName) && (stream->pathName == open_sync_param->logFilePathName)) { TRACE("pathname already exist"); - rc = SA_AIS_ERR_INVALID_PARAM; + ais_rc = SA_AIS_ERR_INVALID_PARAM; goto done; } } @@ -922,7 +922,7 @@ SaAisErrorT create_new_app_stream(lgsv_stream_open_req_t *open_sync_param, str_name = osaf_extended_name_borrow(&open_sync_param->lstr_name); if (strncmp(dnPrefix, str_name, strlen(dnPrefix)) != 0) { TRACE("'%s' is not a valid stream name => invalid param", str_name); - rc = SA_AIS_ERR_INVALID_PARAM; + ais_rc = SA_AIS_ERR_INVALID_PARAM; goto done; } @@ -933,13 +933,13 @@ SaAisErrorT create_new_app_stream(lgsv_stream_open_req_t *open_sync_param, ((open_sync_param->maxLogRecordSize < SA_LOG_MIN_RECORD_SIZE) || (open_sync_param->maxLogRecordSize > logMaxLogrecsize_conf))) { TRACE("maxLogRecordSize is invalid"); - rc = SA_AIS_ERR_INVALID_PARAM; + ais_rc = SA_AIS_ERR_INVALID_PARAM; goto done; } *o_stream = log_stream_new(str_name, STREAM_NEW); if (*o_stream == nullptr) { - rc = SA_AIS_ERR_NO_MEMORY; + ais_rc = SA_AIS_ERR_NO_MEMORY; goto done; } @@ -951,16 +951,19 @@ SaAisErrorT create_new_app_stream(lgsv_stream_open_req_t *open_sync_param, twelveHourModeFlag, 0, *o_stream); // output if (err == -1) { log_stream_delete(o_stream); - rc = SA_AIS_ERR_NO_MEMORY; + ais_rc = SA_AIS_ERR_NO_MEMORY; goto done; } - rc = lgs_create_appstream_rt_object(*o_stream); - if (rc != SA_AIS_OK) log_stream_delete(o_stream); + ais_rc = lgs_create_appstream_rt_object(*o_stream); + if (ais_rc != SA_AIS_OK) log_stream_delete(o_stream); + + TRACE("%s: lgs_create_appstream_rt_object return, %s", + __FUNCTION__, saf_error(ais_rc)); done: TRACE_LEAVE(); - return rc; + return ais_rc; } /** @@ -1069,15 +1072,7 @@ static uint32_t proc_stream_open_msg(lgs_cb_t *cb, lgsv_lgs_evt_t *evt) { } } } else { - /* Stream does not exist */ - - // This check is to avoid the client getting SA_AIS_BAD_OPERATION - // as there is no IMM OI implementer set. - if (cb->immOiHandle == 0) { - TRACE("IMM service unavailable, open stream failed"); - ais_rv = SA_AIS_ERR_TRY_AGAIN; - goto snd_rsp; - } + /* Stream does not exist. Create a new stream */ /* * Check if the stream is in the list of stream objects @@ -1125,6 +1120,11 @@ static uint32_t proc_stream_open_msg(lgs_cb_t *cb, lgsv_lgs_evt_t *evt) { if (ais_rv != SA_AIS_OK) { TRACE("%s create_new_app_stream Fail \"%s\"", __FUNCTION__, saf_error(ais_rv)); + if (ais_rv == SA_AIS_ERR_BAD_HANDLE) { + // This means that the stream RT object could not be created because + // of a bad OI handle. Change to TRY AGAIN in the reply to the agent + ais_rv = SA_AIS_ERR_TRY_AGAIN; + } goto snd_rsp; } } @@ -1218,7 +1218,7 @@ static uint32_t proc_stream_close_msg(lgs_cb_t *cb, lgsv_lgs_evt_t *evt) { // This check is to avoid the client getting SA_AIS_BAD_OPERATION // as there is no IMM OI implementer set. if ((stream->streamType == STREAM_TYPE_APPLICATION_RT) && - (cb->immOiHandle == 0)) { + (lgsGetOiHandle() == 0)) { TRACE("IMM service unavailable, close stream failed"); ais_rc = SA_AIS_ERR_TRY_AGAIN; goto snd_rsp; diff --git a/src/log/logd/lgs_imm.cc b/src/log/logd/lgs_imm.cc index 60870fa17..db95b8522 100644 --- a/src/log/logd/lgs_imm.cc +++ b/src/log/logd/lgs_imm.cc @@ -31,6 +31,8 @@ #define _GNU_SOURCE #endif +#include "log/logd/lgs_imm.h" + #include <stdio.h> #include <errno.h> #include <string.h> @@ -46,6 +48,7 @@ #include "log/logd/lgs_recov.h" #include "log/logd/lgs_config.h" #include "log/logd/lgs_dest.h" +#include "log/logd/lgs_oi_admin.h" #include "base/saf_error.h" #include "lgs_mbcsv_v1.h" @@ -58,11 +61,6 @@ * ---------------- */ -/* Used for protecting global imm OI handle and selection object during - * initialize of OI - */ -pthread_mutex_t lgs_OI_init_mutex = PTHREAD_MUTEX_INITIALIZER; - /* Used for checkpointing time when files are closed */ static time_t chkp_file_close_time = 0; @@ -75,13 +73,6 @@ static const char *log_file_format[] = { DEFAULT_ALM_NOT_FORMAT_EXP, DEFAULT_ALM_NOT_FORMAT_EXP, DEFAULT_APP_SYS_FORMAT_EXP, DEFAULT_APP_SYS_FORMAT_EXP}; -static const SaImmOiImplementerNameT implementerName = - const_cast<SaImmOiImplementerNameT>("safLogService"); -static const SaImmClassNameT logConfig_str = - const_cast<SaImmClassNameT>("OpenSafLogConfig"); -static const SaImmClassNameT streamConfig_str = - const_cast<SaImmClassNameT>("SaLogStreamConfig"); - // The list contains path-file names when validating its name in CCB completed // callback. This help for log service prevents that creating any streams // in same CCB with duplicated log path-file name @@ -118,7 +109,10 @@ static void report_oi_error(SaImmOiHandleT immOiHandle, SaImmOiCcbIdT ccbId, va_end(ap); TRACE("%s", err_str); - (void)saImmOiCcbSetErrorString(immOiHandle, ccbId, err_str); + if (saImmOiCcbSetErrorString(immOiHandle, ccbId, err_str) == + SA_AIS_ERR_BAD_HANDLE) { + lgsOiCreateBackground(); + } } static void report_om_error(SaImmOiHandleT immOiHandle, @@ -137,8 +131,11 @@ static void report_om_error(SaImmOiHandleT immOiHandle, va_end(ap); TRACE("%s", ao_err_string); - (void)saImmOiAdminOperationResult_o2(immOiHandle, invocation, - SA_AIS_ERR_INVALID_PARAM, ao_err_params); + if (saImmOiAdminOperationResult_o2(immOiHandle, invocation, + SA_AIS_ERR_INVALID_PARAM, + ao_err_params) == SA_AIS_ERR_BAD_HANDLE) { + lgsOiCreateBackground(); + } } /** @@ -409,6 +406,7 @@ static void adminOperationCallback( if (stream->isRtStream != SA_TRUE) { ais_rc = immutil_saImmOiAdminOperationResult(immOiHandle, invocation, SA_AIS_ERR_NOT_SUPPORTED); + // TODO(Lennart) May be relevant to recover OI if BAD HANDLE if (ais_rc != SA_AIS_OK) { LOG_ER("immutil_saImmOiAdminOperationResult failed %s", saf_error(ais_rc)); @@ -448,6 +446,7 @@ static void adminOperationCallback( if (severityFilter == stream->severityFilter) { ais_rc = immutil_saImmOiAdminOperationResult(immOiHandle, invocation, SA_AIS_ERR_NO_OP); + // TODO(Lennart) May be relevant to recover OI if BAD HANDLE if (ais_rc != SA_AIS_OK) { LOG_ER("immutil_saImmOiAdminOperationResult failed %s", saf_error(ais_rc)); @@ -466,12 +465,14 @@ static void adminOperationCallback( const_cast<SaImmAttrNameT>("saLogStreamSeverityFilter"), SA_IMM_ATTR_SAUINT32T, &stream->severityFilter); if (ais_rc != SA_AIS_OK) { - LOG_ER("immutil_update_one_rattr failed %s", saf_error(ais_rc)); + LOG_ER("%s: immutil_update_one_rattr failed %s", __FUNCTION__, + saf_error(ais_rc)); osaf_abort(0); } ais_rc = immutil_saImmOiAdminOperationResult(immOiHandle, invocation, SA_AIS_OK); + // TODO(Lennart) May be relevant to recover OI if BAD HANDLE if (ais_rc != SA_AIS_OK) { LOG_ER("immutil_saImmOiAdminOperationResult failed %s", saf_error(ais_rc)); @@ -2375,11 +2376,12 @@ static SaAisErrorT stream_create_and_configure1( /* Update creation timestamp */ rc = immutil_update_one_rattr( - lgs_cb->immOiHandle, objectName.c_str(), + lgsGetOiHandle(), objectName.c_str(), const_cast<SaImmAttrNameT>("saLogStreamCreationTimestamp"), SA_IMM_ATTR_SATIMET, &(*stream)->creationTimeStamp); if (rc != SA_AIS_OK) { - LOG_ER("immutil_update_one_rattr failed %s", saf_error(rc)); + LOG_ER("%s: immutil_update_one_rattr failed %s", __FUNCTION__, + saf_error(rc)); osaf_abort(0); } @@ -2702,7 +2704,7 @@ static SaAisErrorT rtAttrUpdateCallback(SaImmOiHandleT immOiHandle, /* Handle configuration runtime object */ if (strncmp(objName, LGS_CFG_RUNTIME_OBJECT, strlen(objName)) == 0) { /* Handle Runtome configuration object */ - conf_runtime_obj_hdl(immOiHandle, attributeNames); + conf_runtime_obj_handler(immOiHandle, attributeNames); } else { /* Handle stream object if valid */ @@ -2889,11 +2891,15 @@ static const SaImmOiCallbacksT_2 callbacks = { .saImmOiCcbObjectModifyCallback = ccbObjectModifyCallback, .saImmOiRtAttrUpdateCallback = rtAttrUpdateCallback}; +// Export the callback structure. OI init is done in lgs_oi_admin +const SaImmOiCallbacksT_2* getImmOiCallbacks(void) { + return &callbacks; +} + /** * Retrieve the LOG stream configuration from IMM using the * IMM-OM interface and initialize the corresponding information - * in the LOG control block. Initialize the LOG IMM-OI - * interface. Become class implementer. + * in the LOG control block. * * @param cb[in] control block * @return @@ -2987,20 +2993,6 @@ SaAisErrorT lgs_imm_init_configStreams(lgs_cb_t *cb) { } } - /* 1.Become implementer - * 2.Update creation timestamp for all configure object, must be object - * implementer first 3.Open all streams Config file and log file will be - * created. If this fails we give up without returning any error. A new - * attempt to create the files will be done when trying to write a log - * record to the stream. - */ - ais_rc = - immutil_saImmOiClassImplementerSet(cb->immOiHandle, "SaLogStreamConfig"); - if (ais_rc != SA_AIS_OK) { - LOG_ER("immutil_saImmOiClassImplementerSet failed %s", saf_error(ais_rc)); - osaf_abort(0); - } - // Iterate all existing log streams in cluster. while ((stream = iterate_all_streams(endloop, jstart)) && !endloop) { jstart = SA_FALSE; @@ -3021,7 +3013,7 @@ SaAisErrorT lgs_imm_init_configStreams(lgs_cb_t *cb) { } ais_rc = immutil_update_one_rattr( - cb->immOiHandle, stream->name.c_str(), + lgsGetOiHandle(), stream->name.c_str(), const_cast<SaImmAttrNameT>("saLogStreamCreationTimestamp"), SA_IMM_ATTR_SATIMET, &stream->creationTimeStamp); if (ais_rc != SA_AIS_OK) { @@ -3046,255 +3038,6 @@ done: return ais_rc; } -/** - * Initialize the OI interface and get a selection object. - * Become OI for safLogService if Active - * - * @param immOiHandle[out] - * @param immSelectionObject[out] - */ -void lgs_imm_init_OI_handle(SaImmOiHandleT *immOiHandle, - SaSelectionObjectT *immSelectionObject) { - SaAisErrorT rc; - uint32_t msecs_waited = 0; - - TRACE_ENTER(); - - /* Initialize IMM OI service */ - SaVersionT imm_version = kImmVersion; - rc = saImmOiInitialize_2(immOiHandle, &callbacks, &imm_version); - while ((rc == SA_AIS_ERR_TRY_AGAIN) && - (msecs_waited < max_waiting_time_60s)) { - usleep(sleep_delay_ms * 1000); - msecs_waited += sleep_delay_ms; - imm_version = kImmVersion; - rc = saImmOiInitialize_2(immOiHandle, &callbacks, &imm_version); - } - if (rc != SA_AIS_OK) { - lgs_exit("saImmOiInitialize_2 failed", SA_AMF_COMPONENT_RESTART); - } - - /* Get selection object for event handling */ - msecs_waited = 0; - rc = saImmOiSelectionObjectGet(*immOiHandle, immSelectionObject); - while ((rc == SA_AIS_ERR_TRY_AGAIN) && - (msecs_waited < max_waiting_time_10s)) { - usleep(sleep_delay_ms * 1000); - msecs_waited += sleep_delay_ms; - rc = saImmOiSelectionObjectGet(*immOiHandle, immSelectionObject); - } - if (rc != SA_AIS_OK) { - lgs_exit("saImmOiSelectionObjectGet failed", SA_AMF_COMPONENT_RESTART); - } - - TRACE_LEAVE2("rc: %s", saf_error(rc)); -} - -/** - * Does the sequence of setting an implementer name and class implementer - * - * @param immOiHandle[in] - * @return SaAisErrorT - */ -static SaAisErrorT imm_impl_set_sequence( - SaImmOiHandleT *immOiHandle, SaSelectionObjectT *immSelectionObject) { - SaAisErrorT rc = SA_AIS_OK; - uint32_t msecs_waited = 0; - - TRACE_ENTER(); - - for (;;) { - if (msecs_waited >= max_waiting_time_60s) { - TRACE("Timeout in imm_impl_set_sequence"); - goto done; - } - - /* Become object implementer - */ - rc = saImmOiImplementerSet(*immOiHandle, implementerName); - while (((rc == SA_AIS_ERR_TRY_AGAIN) || (rc == SA_AIS_ERR_EXIST)) && - (msecs_waited < max_waiting_time_60s)) { - usleep(sleep_delay_ms * 1000); - msecs_waited += sleep_delay_ms; - rc = saImmOiImplementerSet(*immOiHandle, implementerName); - } - if (rc == SA_AIS_ERR_BAD_HANDLE || rc == SA_AIS_ERR_TIMEOUT) { - LOG_WA("saImmOiImplementerSet returned %s", saf_error(rc)); - usleep(sleep_delay_ms * 1000); - msecs_waited += sleep_delay_ms; - saImmOiFinalize(*immOiHandle); - *immOiHandle = 0; - *immSelectionObject = -1; - lgs_imm_init_OI_handle(immOiHandle, immSelectionObject); - continue; - } - if (rc != SA_AIS_OK) { - TRACE("saImmOiImplementerSet failed %s", saf_error(rc)); - goto done; - } - - /* - * Become class implementer for the OpenSafLogConfig class if it exists - * Become class implementer for the SaLogStreamConfig class - */ - if (true == *static_cast<const bool *>( - lgs_cfg_get(LGS_IMM_LOG_OPENSAFLOGCONFIG_CLASS_EXIST))) { - rc = saImmOiClassImplementerSet(*immOiHandle, logConfig_str); - while (((rc == SA_AIS_ERR_TRY_AGAIN) || (rc == SA_AIS_ERR_EXIST)) && - (msecs_waited < max_waiting_time_60s)) { - usleep(sleep_delay_ms * 1000); - msecs_waited += sleep_delay_ms; - rc = saImmOiClassImplementerSet(*immOiHandle, logConfig_str); - } - if (rc == SA_AIS_ERR_BAD_HANDLE || rc == SA_AIS_ERR_TIMEOUT) { - LOG_WA("saImmOiClassImplementerSet returned %s", saf_error(rc)); - usleep(sleep_delay_ms * 1000); - msecs_waited += sleep_delay_ms; - saImmOiFinalize(*immOiHandle); - *immOiHandle = 0; - *immSelectionObject = -1; - lgs_imm_init_OI_handle(immOiHandle, immSelectionObject); - continue; - } - if (rc != SA_AIS_OK) { - TRACE("saImmOiClassImplementerSet OpenSafLogConfig failed %s", - saf_error(rc)); - goto done; - } - } - - rc = saImmOiClassImplementerSet(*immOiHandle, streamConfig_str); - while (((rc == SA_AIS_ERR_TRY_AGAIN) || (rc == SA_AIS_ERR_EXIST)) && - (msecs_waited < max_waiting_time_60s)) { - usleep(sleep_delay_ms * 1000); - msecs_waited += sleep_delay_ms; - rc = saImmOiClassImplementerSet(*immOiHandle, streamConfig_str); - } - if (rc == SA_AIS_ERR_BAD_HANDLE || rc == SA_AIS_ERR_TIMEOUT) { - LOG_WA("saImmOiClassImplementerSet returned %s", saf_error(rc)); - usleep(sleep_delay_ms * 1000); - msecs_waited += sleep_delay_ms; - saImmOiFinalize(*immOiHandle); - *immOiHandle = 0; - *immSelectionObject = -1; - lgs_imm_init_OI_handle(immOiHandle, immSelectionObject); - continue; - } - if (rc != SA_AIS_OK) { - TRACE("saImmOiClassImplementerSet SaLogStreamConfig failed %s", - saf_error(rc)); - goto done; - } - break; - } - -done: - TRACE_LEAVE2("rc: %s", saf_error(rc)); - return rc; -} - -/** - * Set implementer name and become class implementer. - * This function will block until done. - * - * @param cb - */ -void lgs_imm_impl_set(SaImmOiHandleT *immOiHandle, - SaSelectionObjectT *immSelectionObject) { - SaAisErrorT rc = SA_AIS_OK; - - TRACE_ENTER(); - - rc = imm_impl_set_sequence(immOiHandle, immSelectionObject); - if (rc != SA_AIS_OK) { - lgs_exit("Becoming OI implementer failed", SA_AMF_COMPONENT_RESTART); - } - - TRACE_LEAVE(); -} - -/** - * Thread - * Restore object and class implementer/applier. - * - * @param _cb[in] - */ -static void *imm_impl_init_thread(void *_cb) { - lgs_cb_t *cb = static_cast<lgs_cb_t *>(_cb); - SaSelectionObjectT immSelectionObject = 0; - SaImmOiHandleT immOiHandle = 0; - SaAisErrorT rc = SA_AIS_OK; - - TRACE_ENTER(); - - /* Initialize handles and become implementer */ - lgs_imm_init_OI_handle(&immOiHandle, &immSelectionObject); - rc = imm_impl_set_sequence(&immOiHandle, &immSelectionObject); - if (rc != SA_AIS_OK) { - lgs_exit("Becoming OI implementer failed", SA_AMF_COMPONENT_RESTART); - } - - /* Store handle and selection object. - * Protect if the poll loop in main is released during the storage - * sequence. - */ - osaf_mutex_lock_ordie(&lgs_OI_init_mutex); - cb->immSelectionObject = immSelectionObject; - cb->immOiHandle = immOiHandle; - osaf_mutex_unlock_ordie(&lgs_OI_init_mutex); - - /* Activate the poll loop in main() - * This will reinstall IMM poll event handling - */ - lgsv_lgs_evt_t *lgsv_evt; - lgsv_evt = static_cast<lgsv_lgs_evt_t *>(calloc(1, sizeof(lgsv_lgs_evt_t))); - osafassert(lgsv_evt); - lgsv_evt->evt_type = LGSV_EVT_NO_OP; - if (m_NCS_IPC_SEND(&lgs_mbx, lgsv_evt, LGS_IPC_PRIO_CTRL_MSGS) != - NCSCC_RC_SUCCESS) { - LOG_WA("imm_reinit_thread failed to send IPC message to main thread"); - /* - * Se no reason why this would happen. But if it does at least there - * is something in the syslog. The main thread should still pick up - * the new imm FD when there is a healthcheck, but it could take - *minutes. - */ - free(lgsv_evt); - } - - TRACE_LEAVE(); - return NULL; -} - -/** - * In a separate thread: - * Initiate IMM OI handle and selection object - * Create imm implementer for log IMM objects - * When complete: - * Store the new handle and selection object in the cb store. - * Activate the poll loop in main() by sending an empty mailbox message - * - * @param cb[out] - */ -void lgs_imm_impl_reinit_nonblocking(lgs_cb_t *cb) { - pthread_t thread; - pthread_attr_t attr; - - TRACE_ENTER(); - - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - - if (pthread_create(&thread, &attr, imm_impl_init_thread, cb) != 0) { - LOG_ER("pthread_create FAILED: %s", strerror(errno)); - exit(EXIT_FAILURE); - } - - pthread_attr_destroy(&attr); - - TRACE_LEAVE(); -} - /****************************************************************************** * Functions used for recovery handling ******************************************************************************/ @@ -3403,12 +3146,16 @@ void lgs_delete_one_stream_object(const std::string &name_str) { osaf_extended_name_lend(name_str.c_str(), &object_name); /* and delete the object */ - ais_rc = immutil_saImmOiRtObjectDelete(lgs_cb->immOiHandle, &object_name); + ais_rc = immutil_saImmOiRtObjectDelete(lgsGetOiHandle(), &object_name); if (ais_rc == SA_AIS_OK) { TRACE("%s Object \"%s\" deleted", __FUNCTION__, name_str.c_str()); + } else if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + LOG_NO("%s: saImmOiRtObjectDelete() Fail, %s", __FUNCTION__, + saf_error(ais_rc)); + lgsOiCreateBackground(); } else { - LOG_WA("%s saImmOiRtObjectDelete for \"%s\" FAILED %d", __FUNCTION__, - name_str.c_str(), ais_rc); + LOG_WA("%s saImmOiRtObjectDelete for \"%s\" FAILED %s", __FUNCTION__, + name_str.c_str(), saf_error(ais_rc)); } } @@ -3444,12 +3191,16 @@ void lgs_cleanup_abandoned_streams() { /* Copy name to a SaNameT */ osaf_extended_name_lend(name_str, &object_name); /* and delete the object */ - ais_rc = immutil_saImmOiRtObjectDelete(lgs_cb->immOiHandle, &object_name); + ais_rc = immutil_saImmOiRtObjectDelete(lgsGetOiHandle(), &object_name); if (ais_rc == SA_AIS_OK) { TRACE("\tObject \"%s\" deleted", name_str); + } else if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + LOG_WA("%s saImmOiRtObjectDelete for \"%s\" FAILED %s", __FUNCTION__, + name_str, saf_error(ais_rc)); + lgsOiCreateBackground(); } else { - LOG_WA("%s saImmOiRtObjectDelete for \"%s\" FAILED %d", __FUNCTION__, - name_str, ais_rc); + LOG_WA("%s saImmOiRtObjectDelete for \"%s\" FAILED %s", __FUNCTION__, + name_str, saf_error(ais_rc)); } } else { /* Should never happen! */ diff --git a/src/log/logd/lgs_imm.h b/src/log/logd/lgs_imm.h new file mode 100644 index 000000000..195d09380 --- /dev/null +++ b/src/log/logd/lgs_imm.h @@ -0,0 +1,53 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2008 The OpenSAF Foundation + * Copyright Ericsson AB 2008, 2018 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#ifndef LGS_IMM_H +#define LGS_IMM_H + +#include <time.h> +#include <string> + +#include "ais/include/saImmOi.h" +#include "ais/include/saImmOm.h" + +#include "log/logd/lgs_cb.h" + +// Get the callback structure used when initializing an OI handle +const SaImmOiCallbacksT_2* getImmOiCallbacks(void); + +void logRootDirectory_filemove(const std::string &new_logRootDirectory, + const std::string &old_logRootDirectory, + time_t *cur_time_in); +void logDataGroupname_fileown(const char *new_logDataGroupname); + +SaAisErrorT lgs_imm_init_configStreams(lgs_cb_t *cb); + +// Functions for recovery handling +void lgs_cleanup_abandoned_streams(); +void lgs_delete_one_stream_object(const std::string &name_str); +void lgs_search_stream_objects(); +SaUint32T *lgs_get_scAbsenceAllowed_attr(SaUint32T *attr_val); +int lgs_get_streamobj_attr(SaImmAttrValuesT_2 ***attrib_out, + const std::string &object_name, + SaImmHandleT *immOmHandle); +int lgs_free_streamobj_attr(SaImmHandleT immHandle); + + + +#endif /* LGS_IMM_H */ + diff --git a/src/log/logd/lgs_main.cc b/src/log/logd/lgs_main.cc index 988ebdd78..9767fe00d 100644 --- a/src/log/logd/lgs_main.cc +++ b/src/log/logd/lgs_main.cc @@ -33,16 +33,18 @@ #include "nid/agent/nid_api.h" #include "base/ncs_main_papi.h" #include "base/osaf_time.h" - -#include "log/logd/lgs.h" -#include "log/logd/lgs_file.h" #include "base/osaf_utility.h" #include "base/hash.h" -#include "lgs_recov.h" #include "osaf/immutil/immutil.h" + +#include "log/logd/lgs.h" +#include "log/logd/lgs_file.h" +#include "log/logd/lgs_recov.h" #include "lgs_clm.h" #include "log/logd/lgs_dest.h" #include "log/logd/lgs_amf.h" +#include "log/logd/lgs_oi_admin.h" +#include "log/logd/lgs_imm.h" /* ======================================================================== @@ -383,26 +385,21 @@ uint32_t initialize_for_assignment(lgs_cb_t *cb, SaAmfHAStateT ha_state) { if (cb->fully_initialized || ha_state == SA_AMF_HA_QUIESCED) goto done; cb->ha_state = ha_state; - /* Initialize IMM OI handle and selection object */ - lgs_imm_init_OI_handle(&cb->immOiHandle, &cb->immSelectionObject); - - TRACE("IMM init done: cb->immOiHandle = %lld", cb->immOiHandle); - /* Initialize log configuration * Must be done after IMM OI is initialized */ - lgs_cfg_init(cb->immOiHandle, cb->ha_state); + lgs_cfg_init(); lgs_trace_config(); /* Show all configuration in TRACE */ - /* Show some configurtion info in sysylog */ - logsv_root_dir = - static_cast<const char *>(lgs_cfg_get(LGS_IMM_LOG_ROOT_DIRECTORY)); - logsv_data_groupname = - static_cast<const char *>(lgs_cfg_get(LGS_IMM_DATA_GROUPNAME)); + /* Show some configuration info in sysylog */ + logsv_root_dir = static_cast<const char *> + (lgs_cfg_get(LGS_IMM_LOG_ROOT_DIRECTORY)); + logsv_data_groupname = static_cast<const char *> + (lgs_cfg_get(LGS_IMM_DATA_GROUPNAME)); LOG_NO("LOG root directory is: \"%s\"", logsv_root_dir); LOG_NO("LOG data group is: \"%s\"", logsv_data_groupname); - vdest = reinterpret_cast<const std::vector<std::string> *>( - lgs_cfg_get(LGS_IMM_LOG_RECORD_DESTINATION_CONFIGURATION)); + vdest = reinterpret_cast<const std::vector<std::string> *> + (lgs_cfg_get(LGS_IMM_LOG_RECORD_DESTINATION_CONFIGURATION)); osafassert(vdest != nullptr); if (vdest->size() > 0) { CfgDestination(*vdest, ModifyType::kAdd); @@ -445,20 +442,15 @@ uint32_t initialize_for_assignment(lgs_cb_t *cb, SaAmfHAStateT ha_state) { } if (ha_state == SA_AMF_HA_ACTIVE) { - /* Become OI. We will be blocked here until done */ - lgs_imm_impl_set(&cb->immOiHandle, &cb->immSelectionObject); - conf_runtime_obj_create(cb->immOiHandle); + // Request an Object Implementer, a configuration object handler and + // an applier for opensafNetworkName change surveillance + lgsOiCreateSynchronous(); + conf_runtime_obj_create(lgsGetOiHandle()); lgs_start_gcfg_applier(); - /* Create streams that has configuration objects and become - * class implementer for the SaLogStreamConfig class - */ - - /* Note1: cb->immOiHandle is set in lgs_imm_init() - * Note2: cb->logsv_root_dir must be set - */ + // Create streams that has configuration objects if (lgs_imm_init_configStreams(cb) != SA_AIS_OK) { - LOG_ER("lgs_imm_create_configStream FAILED"); + LOG_ER("lgs_imm_create_configStreams FAILED"); rc = NCSCC_RC_FAILURE; goto done; } @@ -479,7 +471,7 @@ done: */ int main(int argc, char *argv[]) { NCS_SEL_OBJ mbx_fd; - SaAisErrorT error = SA_AIS_OK; + SaAisErrorT ais_rc = SA_AIS_OK; uint32_t rc; int term_fd; @@ -519,7 +511,7 @@ int main(int argc, char *argv[]) { fds[FD_AMF].events = POLLIN; fds[FD_MBX].fd = mbx_fd.rmv_obj; fds[FD_MBX].events = POLLIN; - fds[FD_IMM].fd = lgs_cb->immSelectionObject; + fds[FD_IMM].fd = lgsGetOiSelectionObject(); fds[FD_IMM].events = POLLIN; lgs_cb->clmSelectionObject = lgs_cb->clm_init_sel_obj.rmv_obj; @@ -539,18 +531,16 @@ int main(int argc, char *argv[]) { fds[FD_CLM].fd = lgs_cb->clmSelectionObject; fds[FD_CLM].events = POLLIN; - /* Protect since the reinit thread may be in the process of - * changing the values - */ - osaf_mutex_lock_ordie(&lgs_OI_init_mutex); - if (lgs_cb->immOiHandle != 0) { - fds[FD_IMM].fd = lgs_cb->immSelectionObject; + // Adds or removes the IMM fd depending on if there is an OI or not + SaImmOiHandleT oi_handle; + SaSelectionObjectT oi_selection_object = lgsGetOiFdsParams(&oi_handle); + if (oi_handle != 0) { + fds[FD_IMM].fd = oi_selection_object; fds[FD_IMM].events = POLLIN; nfds = FD_IMM + 1; } else { nfds = FD_IMM; } - osaf_mutex_unlock_ordie(&lgs_OI_init_mutex); int ret = poll(fds, nfds, -1); @@ -567,9 +557,9 @@ int main(int argc, char *argv[]) { if (fds[FD_AMF].revents & POLLIN) { if (lgs_cb->amf_hdl != 0) { - if ((error = saAmfDispatch(lgs_cb->amf_hdl, SA_DISPATCH_ALL)) != + if ((ais_rc = saAmfDispatch(lgs_cb->amf_hdl, SA_DISPATCH_ALL)) != SA_AIS_OK) { - LOG_ER("saAmfDispatch failed: %u", error); + LOG_ER("saAmfDispatch failed: %u", ais_rc); break; } } else { @@ -593,9 +583,9 @@ int main(int argc, char *argv[]) { if (fds[FD_CLM].revents & POLLIN) { if (lgs_cb->clm_hdl != 0) { - if ((error = saClmDispatch(lgs_cb->clm_hdl, SA_DISPATCH_ALL)) != + if ((ais_rc = saClmDispatch(lgs_cb->clm_hdl, SA_DISPATCH_ALL)) != SA_AIS_OK) { - LOG_ER("saClmDispatch failed: %u", error); + LOG_ER("saClmDispatch failed: %u", ais_rc); break; } } else { @@ -629,40 +619,16 @@ int main(int argc, char *argv[]) { if (fds[FD_MBX].revents & POLLIN) lgs_process_mbx(&lgs_mbx); - if (lgs_cb->immOiHandle && fds[FD_IMM].revents & POLLIN) { - error = saImmOiDispatch(lgs_cb->immOiHandle, SA_DISPATCH_ALL); - - /* - * BAD_HANDLE is interpreted as an IMM service restart. Try - * reinitialize the IMM OI API in a background thread and let - * this thread do business as usual especially handling write - * requests. - * - * All other errors are treated as non-recoverable (fatal) and will - * cause an exit of the process. - */ - if (error == SA_AIS_ERR_BAD_HANDLE) { - TRACE("saImmOiDispatch returned BAD_HANDLE"); - - /* - * Invalidate the IMM OI handle, this info is used in other - * locations. E.g. giving TRY_AGAIN responses to a create and - * close app stream requests. That is needed since the IMM OI - * is used in context of these functions. - * - * Also closing the handle. Finalize is ok with a bad handle - * that is bad because it is stale and this actually clears - * the handle from internal agent structures. In any case - * we ignore the return value from Finalize here. - */ - saImmOiFinalize(lgs_cb->immOiHandle); - lgs_cb->immOiHandle = 0; - lgs_cb->immSelectionObject = -1; - - /* Initiate IMM reinitializtion in the background */ - lgs_imm_impl_reinit_nonblocking(lgs_cb); - } else if (error != SA_AIS_OK) { - LOG_ER("saImmOiDispatch FAILED: %u", error); + if (fds[FD_IMM].revents & POLLIN) { + ais_rc = saImmOiDispatch(lgsGetOiHandle(), SA_DISPATCH_ALL); + + if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + LOG_NO("saImmOiDispatch returned BAD_HANDLE"); + + // Request creation of a new OI + lgsOiCreateBackground(); + } else if (ais_rc != SA_AIS_OK) { + LOG_ER("saImmOiDispatch FAILED: %u", ais_rc); break; } } diff --git a/src/log/logd/lgs_mbcsv_v2.cc b/src/log/logd/lgs_mbcsv_v2.cc index 9afc30155..63807e1b7 100644 --- a/src/log/logd/lgs_mbcsv_v2.cc +++ b/src/log/logd/lgs_mbcsv_v2.cc @@ -21,9 +21,10 @@ * version 2 check-pointing. See also lgs_mbcsv_v2.h */ -#include "lgs_config.h" -#include "lgs_mbcsv_v2.h" -#include "lgs_mbcsv.h" +#include "log/logd/lgs_config.h" +#include "log/logd/lgs_imm.h" +#include "log/logd/lgs_mbcsv_v2.h" +#include "log/logd/lgs_mbcsv.h" /**************************************************************************** * Name : ckpt_proc_lgs_cfg diff --git a/src/log/logd/lgs_mbcsv_v3.cc b/src/log/logd/lgs_mbcsv_v3.cc index f9836c8af..75fd98dba 100644 --- a/src/log/logd/lgs_mbcsv_v3.cc +++ b/src/log/logd/lgs_mbcsv_v3.cc @@ -21,9 +21,10 @@ * version 3 check-pointing. See also lgs_mbcsv_v3.h */ -#include "lgs_mbcsv_v3.h" -#include "lgs_mbcsv_v2.h" -#include "lgs_mbcsv.h" +#include "log/logd/lgs_imm.h" +#include "log/logd/lgs_mbcsv_v3.h" +#include "log/logd/lgs_mbcsv_v2.h" +#include "log/logd/lgs_mbcsv.h" /**************************************************************************** * Name : ckpt_proc_lgs_cfg diff --git a/src/log/logd/lgs_mbcsv_v5.cc b/src/log/logd/lgs_mbcsv_v5.cc index 75d627688..d219f8a5b 100644 --- a/src/log/logd/lgs_mbcsv_v5.cc +++ b/src/log/logd/lgs_mbcsv_v5.cc @@ -22,8 +22,9 @@ * V5 applies to log server configuration check-pointing */ -#include "lgs_mbcsv_v5.h" #include "log/logd/lgs_dest.h" +#include "log/logd/lgs_imm.h" +#include "log/logd/lgs_mbcsv_v5.h" /**************************************************************************** * Name : ckpt_proc_lgs_cfg diff --git a/src/log/logd/lgs_oi_admin.cc b/src/log/logd/lgs_oi_admin.cc new file mode 100644 index 000000000..8b899219e --- /dev/null +++ b/src/log/logd/lgs_oi_admin.cc @@ -0,0 +1,464 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2008 The OpenSAF Foundation + * Copyright Ericsson AB 2018 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "log/logd/lgs_oi_admin.h" + +#include <atomic> +#include <thread> + +#include "ais/include/saAis.h" +#include "ais/include/saImmOi.h" +#include "base/logtrace.h" +#include "base/mutex.h" +#include "base/saf_error.h" +#include "base/time.h" + +#include "log/logd/lgs.h" +#include "log/logd/lgs_imm.h" +#include "log/logd/lgs_config.h" +#include "log/logd/lgs_util.h" + +// Note: Always use the set and get function with these varibales +static base::Mutex lgs_OI_init_mutex; +static SaImmOiHandleT oi_handle_ = 0; +static SaSelectionObjectT oi_selection_object_ = -1; + +static const SaImmOiImplementerNameT kImplementerName = + const_cast<SaImmOiImplementerNameT>("safLogService"); +// The log server is OI for the following configuration classes +static const SaImmClassNameT kLogConfigurationClass = + const_cast<SaImmClassNameT>("OpenSafLogConfig"); +static const SaImmClassNameT kLogStreamClass = + const_cast<SaImmClassNameT>("SaLogStreamConfig"); + +// Max time for creation of OI +static const uint64_t kOiCreationTimeout = 90000; // 1.5 minutes +// Max time for getting an OI handle +static const uint64_t kOiHandleRequestTimeout = 60000; // 1 minute +// Max time for other APIs in the OI creation sequence +static const uint64_t kOiRequestTimeout = 10000; // 10 sec +// Max time to wait for the thread to stop if OI create stop is requested +static const uint64_t kOiWaitStopTimeout = 11000; // 11 sec +// Delay time used in timed out wait loops +static const timespec kOiTryAgainDelay = base::kOneHundredMilliseconds; + +// Internal help functions +// --------------------------------- + +static void setProtectedGlobals(SaImmOiHandleT oi_handle, + SaSelectionObjectT oi_selction_object) { + TRACE_ENTER(); + base::Lock protect_globals(lgs_OI_init_mutex); + oi_handle_ = oi_handle; + oi_selection_object_ = oi_selction_object; +} + +static SaAisErrorT finalizeOi(SaImmOiHandleT oi_handle) { + TRACE_ENTER(); + SaAisErrorT ais_rc = SA_AIS_OK; + base::Timer try_again_timer(kOiRequestTimeout); + while (try_again_timer.is_timeout() == false) { + ais_rc = saImmOiFinalize(oi_handle); + if (ais_rc == SA_AIS_ERR_TRY_AGAIN) { + base::Sleep(kOiTryAgainDelay); + continue; + } else if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + // If we get BAD HANDLE the OI is already finalized + ais_rc = SA_AIS_OK; + break; + } else if (ais_rc != SA_AIS_OK) { + TRACE("%s: saImmOiFinalize() Fail, %s", __FUNCTION__, saf_error(ais_rc)); + } + break; + } + if ((try_again_timer.is_timeout() == true) && + (ais_rc == SA_AIS_ERR_TRY_AGAIN)) { + LOG_NO("%s: saImmOiFinalize() Fail, TRY AGAIN timeout", __FUNCTION__); + } + + return ais_rc; +} + +// IMM API help function used in the OI creation thread +// Communication flag read in OI thread and set by the lgs_OI_delete() and +// create functions. When this flag is true the thread shall stop and return +// as soon as possible +// Note1: All of these functions will stop internal loops and return if +// the global atomic flag oi_stop == true +// Note2: Only the function used for stopping the OI is allowed to write this +// flag +static std::atomic<bool> stop_oi_create(false); +// Flag indicating whether the OI creation thread is running or not +static std::atomic<bool> oi_create_is_executing(false); + +// Output: out_oi_handle. Set to 0 if fail +static SaAisErrorT initOiHandle(SaImmOiHandleT *out_oi_handle) { + TRACE_ENTER(); + SaAisErrorT ais_rc = SA_AIS_OK; + const SaImmOiCallbacksT_2 *callbacks = getImmOiCallbacks(); + base::Timer try_again_timer(kOiHandleRequestTimeout); + while (try_again_timer.is_timeout() == false) { + if (stop_oi_create) { + ais_rc = SA_AIS_OK; + break; + } + SaVersionT imm_version = kImmVersion; + ais_rc = saImmOiInitialize_2(out_oi_handle, callbacks, &imm_version); + if (ais_rc == SA_AIS_ERR_TRY_AGAIN) { + base::Sleep(kOiTryAgainDelay); + continue; + } + if (ais_rc != SA_AIS_OK) { + LOG_WA("%s: saImmOiInitialize_2() Fail, %s", __FUNCTION__, + saf_error(ais_rc)); + *out_oi_handle = 0; + } + break; + } + if ((try_again_timer.is_timeout() == true) && + (ais_rc == SA_AIS_ERR_TRY_AGAIN)) { + LOG_WA("%s: saImmOiInitialize_2() Fail, TRY AGAIN timeout", __FUNCTION__); + *out_oi_handle = 0; + } + + return ais_rc; +} + +// Return -1 if Fail +// out_ais_rc is set to return code from saImmOiSelectionObjectGet() +static SaSelectionObjectT requestOiSelectionObject(SaImmOiHandleT oi_handle, + SaAisErrorT* out_ais_rc) { + TRACE_ENTER(); + SaSelectionObjectT oi_selection_object = -1; + SaAisErrorT ais_rc = SA_AIS_OK; + base::Timer try_again_timer(kOiRequestTimeout); + while (try_again_timer.is_timeout() == false) { + if (stop_oi_create) { + oi_selection_object = -1; + break; + } + ais_rc = saImmOiSelectionObjectGet(oi_handle, &oi_selection_object); + if (ais_rc == SA_AIS_ERR_TRY_AGAIN) { + base::Sleep(kOiTryAgainDelay); + continue; + } + if (ais_rc != SA_AIS_OK) { + LOG_WA("%s: saImmOiSelectionObjectGet() Fail, %s", __FUNCTION__, + saf_error(ais_rc)); + oi_selection_object = -1; + } + break; + } + if ((try_again_timer.is_timeout() == true) && + (ais_rc == SA_AIS_ERR_TRY_AGAIN)) { + LOG_WA("%s: saImmOiSelectionObjectGet() Fail, TRY AGAIN timeout", + __FUNCTION__); + oi_selection_object = -1; + } + + *out_ais_rc = ais_rc; + return oi_selection_object; +} + +// Set implementer name in kImplementerName +static SaAisErrorT setOiImplementerName(SaImmOiHandleT oi_handle) { + TRACE_ENTER(); + SaAisErrorT ais_rc = SA_AIS_OK; + base::Timer try_again_timer(kOiRequestTimeout); + while (try_again_timer.is_timeout() == false) { + if (stop_oi_create) { + ais_rc = SA_AIS_OK; + break; + } + ais_rc = saImmOiImplementerSet(oi_handle, kImplementerName); + if (ais_rc == SA_AIS_ERR_TRY_AGAIN) { + base::Sleep(kOiTryAgainDelay); + continue; + } else if (ais_rc == SA_AIS_ERR_EXIST) { + // Should be released. May be temporary. Lets try again + TRACE("%s saImmOiImplementerSet() Fail %s. Try to create again", + __FUNCTION__, saf_error(ais_rc)); + base::Sleep(kOiTryAgainDelay); + continue; + } else if (ais_rc != SA_AIS_OK) { + LOG_WA("%s: saImmOiImplementerSet() Fail, %s", __FUNCTION__, + saf_error(ais_rc)); + break; + } + break; + } + if ((try_again_timer.is_timeout() == true) && + (ais_rc == SA_AIS_ERR_TRY_AGAIN)) { + LOG_WA("%s: saImmOiImplementerSet() Fail, TRY AGAIN timeout", + __FUNCTION__); + } + + TRACE_LEAVE2("ais_rc = %s", saf_error(ais_rc)); + return ais_rc; +} + +static SaAisErrorT setClassImplementer(SaImmOiHandleT oi_handle, + const SaImmClassNameT class_name) { + TRACE_ENTER(); + SaAisErrorT ais_rc = SA_AIS_OK; + base::Timer try_again_timer(kOiRequestTimeout); + while (try_again_timer.is_timeout() == false) { + if (stop_oi_create) { + ais_rc = SA_AIS_OK; + break; + } + ais_rc = saImmOiClassImplementerSet(oi_handle, class_name); + if (ais_rc == SA_AIS_ERR_TRY_AGAIN) { + base::Sleep(kOiTryAgainDelay); + continue; + } + if (ais_rc != SA_AIS_OK) { + LOG_WA("%s: saImmOiClassImplementerSet() Fail, %s", __FUNCTION__, + saf_error(ais_rc)); + } + break; + } + if ((try_again_timer.is_timeout() == true) && + (ais_rc == SA_AIS_ERR_TRY_AGAIN)) { + LOG_WA("%s: saImmOiClassImplementerSet() Fail, TRY AGAIN timeout", + __FUNCTION__); + } + + return ais_rc; +} + +// Activate the poll loop in main() +// This will reinstall IMM poll event handling +static void sendOiCreatedMessageToMainThread() { + TRACE_ENTER(); + // Note: lgsv_evt must be allocated C style since it will be freed in other + // part of the code with free() + lgsv_lgs_evt_t *lgsv_evt; + lgsv_evt = static_cast<lgsv_lgs_evt_t *>(calloc(1, sizeof(lgsv_lgs_evt_t))); + osafassert(lgsv_evt); + + lgsv_evt->evt_type = LGSV_EVT_NO_OP; + if (m_NCS_IPC_SEND(&lgs_mbx, lgsv_evt, LGS_IPC_PRIO_CTRL_MSGS) != + NCSCC_RC_SUCCESS) { + LOG_WA("imm_reinit_thread failed to send IPC message to main thread"); + /* + * Se no reason why this would happen. But if it does at least there + * is something in the syslog. The main thread should still pick up + * the new imm FD when there is a healthcheck, but it could take + *minutes. + */ + free(lgsv_evt); + } +} + +// Creates the log service OI +// stop_oi_create: Used to stop thread execution as soon as possible +// Will stop the thread when set to true. +static void createLogServerOi() { + TRACE_ENTER(); + SaAisErrorT ais_rc = SA_AIS_OK; + SaImmOiHandleT oi_handle = 0; + SaSelectionObjectT oi_selection_object = -1; + base::Timer io_create_timeout(kOiCreationTimeout); + + while (io_create_timeout.is_timeout() == false) { + // OI init sequence + // Start by finalize OI (handle) in case of sequence recovery + // See continue below + finalizeOi(oi_handle); + + // Init OI handle + if (stop_oi_create) break; + ais_rc = initOiHandle(&oi_handle); + if (ais_rc != SA_AIS_OK) { + LOG_NO("%s: initOiHandle() Fail", __FUNCTION__); + oi_handle = 0; + break; + } + + // Set OI implementer name + if (stop_oi_create) break; + ais_rc = setOiImplementerName(oi_handle); + if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + TRACE("%s: setOiImplementerName() Fail %s. Try to recover", + __FUNCTION__, saf_error(ais_rc)); + continue; + } else if (ais_rc != SA_AIS_OK) { + LOG_NO("%s: setOiImplementerName() Fail", __FUNCTION__); + break; + } + + // Become class implementer for log config class + if (stop_oi_create) break; + ais_rc = setClassImplementer(oi_handle, kLogConfigurationClass); + if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + TRACE("%s: setClassImplementer(LogConfigurationClass) Fail %s. " + "Try to recover", __FUNCTION__, saf_error(ais_rc)); + continue; + } else if (ais_rc != SA_AIS_OK) { + LOG_NO("%s: setClassImplementer(LogConfigurationClass) Fail", + __FUNCTION__); + break; + } + + // Become class implementer for stream config class + if (stop_oi_create) break; + ais_rc = setClassImplementer(oi_handle, kLogStreamClass); + if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + TRACE("%s: setClassImplementer(kLogStreamClass) Fail %s. " + "Try to recover", __FUNCTION__, saf_error(ais_rc)); + continue; + } else if (ais_rc != SA_AIS_OK) { + LOG_NO("%s: setClassImplementer(kLogStreamClass) Fail", __FUNCTION__); + break; + } else { + // Get the selection object + oi_selection_object = requestOiSelectionObject(oi_handle, &ais_rc); + if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + TRACE("%s: requestOiSelectionObject() Fail", __FUNCTION__); + } + break; + } + } + + // Creation is done, has failed or is stopped. + if (stop_oi_create) { + TRACE("%s: OI create is stopped", __FUNCTION__); + } else if ((io_create_timeout.is_timeout() == true) && + (ais_rc != SA_AIS_OK)) { + LOG_WA("%s: Fail, OI creation timeout", __FUNCTION__); + // The legacy behavior is to exit log service in this case + lgs_exit("createLogServerOi() failed", SA_AMF_COMPONENT_RESTART); + } else if (ais_rc != SA_AIS_OK) { + LOG_WA("%s: Fail, OI creation error", __FUNCTION__); + // The legacy behavior is to exit log service in this case + lgs_exit("createLogServerOi() failed", SA_AMF_COMPONENT_RESTART); + } + + // Save OI handle and selection object + // Note: This can be done unconditionally here since failing will result in + // a log server restart + setProtectedGlobals(oi_handle, oi_selection_object); +} + +// The thread function for creating a log server OI +// stop_oi_create: Used to stop thread execution as soon as possible +// Will stop the thread when set to true. +static void oiCreationThread() { + TRACE_ENTER(); + createLogServerOi(); + sendOiCreatedMessageToMainThread(); + oi_create_is_executing = false; + return; +} + +// Public functions +//----------------- + +SaImmOiHandleT lgsGetOiHandle() { + base::Lock protect_globals(lgs_OI_init_mutex); + return oi_handle_; +} + +SaSelectionObjectT lgsGetOiSelectionObject() { + base::Lock protect_globals(lgs_OI_init_mutex); + return oi_selection_object_; +} + +SaSelectionObjectT lgsGetOiFdsParams(SaImmOiHandleT* out_oi_handle) { + base::Lock protect_globals(lgs_OI_init_mutex); + *out_oi_handle = oi_handle_; + return oi_selection_object_; +} + +void lgsOiCreateBackground() { + TRACE_ENTER(); + if (oi_create_is_executing == true) { + // Thread is executing. Do nothing + return; + } + + // Make sure that the OI is finalized before creating a new one + finalizeOi(oi_handle_); + setProtectedGlobals(0, -1); + + // Start a OI creation thread + // Note: Throws an exception if "The system is unable to start a new thread" + // Should never happen so it is not handled here (will cause node restart) + oi_create_is_executing = true; + std::thread (&oiCreationThread).detach(); +} + +void lgsOiCreateSynchronous() { + TRACE_ENTER(); + // Note: oi_handle_ can be used unprotected here since we know that there is + // no running oiCreationThread + // There is no meaning to call finalizeOI() since this function only shall be + // used for the first creation of an OI handle during initialization of the + // log service + oi_handle_ = 0; + oi_selection_object_ = -1; + createLogServerOi(); +} + +// Note: In this version this is just call lgsOiCreateBackground(). The reason +// for having this function is to have a place where an implementer set can +// be done instead of creating an OI if the lgsOiStop function is changed +// to just clear the OI. Note that if set fail the OI must be created anyway +// +// This function shall be used only when changing HA state from standby to +// active +void lgsOiStart() { + TRACE_ENTER(); + lgsOiCreateBackground(); +} + +// Note: oi_handle_ and oi_selection_object_ is always set in thread also if +// thread is stopped. Is set to 0 and -1 respectively in all cases except when +// an OI is successfully created +// +// Note: This version will completely finalize the OI. This is probably good +// enough. Clear / Set handling may be a bit faster but Stop / Start of +// OI should be rare. +// If optimization is needed we can do implementer clear here +void lgsOiStop() { + TRACE_ENTER(); + // This is a request for the OI creation thread to stop + stop_oi_create = true; + + // Wait for thread to stop if it is running + base::Timer io_thread_stop_timeout(kOiWaitStopTimeout); + while (io_thread_stop_timeout.is_timeout() != true) { + // Wait for the OI creation thread to stop (return) + // Note: The thread has to be detached so join cannot be used + if (oi_create_is_executing == false) break; + base::Sleep(base::kOneHundredMilliseconds); + } + if ((io_thread_stop_timeout.is_timeout() == true) && + (oi_create_is_executing == true)) { + // The OI creation thread is hanging. Recover by restarting log server + lgs_exit("lgsOiStop: Stopping OI create thread timeout", + SA_AMF_COMPONENT_RESTART); + } + + // Note: oi_handle_ can be used unprotected here since we know that there is + // no running oiCreationThread + finalizeOi(oi_handle_); + setProtectedGlobals(0, -1); + stop_oi_create = false; +} diff --git a/src/log/logd/lgs_oi_admin.h b/src/log/logd/lgs_oi_admin.h new file mode 100644 index 000000000..9ea0893ea --- /dev/null +++ b/src/log/logd/lgs_oi_admin.h @@ -0,0 +1,108 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2008 The OpenSAF Foundation + * Copyright Ericsson AB 2018 - All Rights Reserved. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ + +#include "ais/include/saImmOi.h" + +// Contains administrative handling of the lgs main object implementer (OI) +// Implements functions for initializing an OI in a background thread, +// finalizing the OI and stop an ongoing initialization +// The reason for using a background thread is to not block the log server. +// An initialize request may take up to 1 minute. +// When stopping the OI a running OI initializing thread is stopped and the OI +// is finalized. +// +// Examples of situations when the functions are used: +// Starting log server: OI shall be intialized. This should be done in +// background so that start of log server is not blocked. +// +// Switch/fail over: The to become standby; must give up the OI. The chosen +// way of doing that is to finalize the OI. An alternative +// is to only clear the OI which is no longer used here. +// The to become active; must initialize an OI. An +// alternative is to keep the OI handle and just use the +// set API +// +// BAD HANDLE: IMM may return bad handle when synchronous operations +// are called including the dispatch function. If this +// happen the OI shall be initialized again except in a +// switch/fail over situation where the server shall become +// standby +// +// Note1: The background thread puts a no-operation message in the mailbox +// handled by the main poll loop when OI initialize is done. This is +// needed to handle installing of the IMM fd in the poll loop. +// +// Note2: The previous global immOiHandle and immSelectionObject variables in +// cb struct are removed. Values can be fetched using thread safe get +// functions + +#ifndef LGS_OI_ADMIN_H +#define LGS_OI_ADMIN_H + +// Creates a new OI. Return false if fail. +// Note1: If there is an existing OI it will be deleted and a new is created. +// This means calling IMM finalize, setting oi_handle = 0, +// oi_selection_object = -1 and starting the OI creation thread. The +// OI creation thread will update oi_handle and oi_selection_object with +// new values when creation is done. +// Nothing will be done if OI creation is already ongoing when this +// function is called +// Note2: lgsOiCreateBackground() shall not be used when log server is starting. +// During initialization configuration streams are created. +// When creating a configuration stream cached Rt attributes has to be +// written and this requires a valid OI handle +// Instead use lgsOiCreateSyncronous() +// +// This function shall also be used to restore the OI in case of a BAD HANDLE +// return code when using the OI handle with an OI IMM operation. +// Not thread safe. Shall be used in the log server main thread only +void lgsOiCreateBackground(); + +// Note3: This function shall only be called once and only when the log server +// is initializing +// This function is not thread safe +void lgsOiCreateSynchronous(); + +// Start an OI. +// Shall be used when this server shall become an OI. +// For example when becoming active +// Not thread safe. Shall be used in the log server main thread only +void lgsOiStart(); + +// Stop an existing OI +// To be used when this server no longer shall be an OI. +// For example becoming standby +// Not thread safe. Shall be used in the log server main thread only +void lgsOiStop(); + +// Get both OI handle and selection object. Guarantees that we get a pair +// that is created together. Return -1 if there is no selection object and +// the value of out_oi_handle will be 0 +// Is thread safe +SaSelectionObjectT lgsGetOiFdsParams(SaImmOiHandleT* out_oi_handle); + +// Returns the created OI selection object or -1 if there is none +// Is thread safe +SaSelectionObjectT lgsGetOiSelectionObject(); + +// Returns the created OI handle or 0 if there is no handle +// Is thread safe +SaImmOiHandleT lgsGetOiHandle(); + +#endif /* LGS_OI_ADMIN_H */ + diff --git a/src/log/logd/lgs_recov.cc b/src/log/logd/lgs_recov.cc index dcaad188d..7c0197b7f 100644 --- a/src/log/logd/lgs_recov.cc +++ b/src/log/logd/lgs_recov.cc @@ -15,9 +15,11 @@ * */ -#include "lgs_recov.h" +#include "log/logd/lgs_recov.h" + #include "log/logd/lgs_file.h" #include "log/logd/lgs_filehdl.h" +#include "log/logd/lgs_imm.h" /*** * The following functions are used to handle a list of runtime stream objects diff --git a/src/log/logd/lgs_stream.cc b/src/log/logd/lgs_stream.cc index 0c83e305b..4f50f5a98 100644 --- a/src/log/logd/lgs_stream.cc +++ b/src/log/logd/lgs_stream.cc @@ -30,14 +30,16 @@ #include "log/logd/lgs_stream.h" #include <algorithm> +#include "base/osaf_time.h" +#include "osaf/immutil/immutil.h" + #include "log/logd/lgs.h" #include "log/logd/lgs_config.h" #include "log/logd/lgs_file.h" #include "log/logd/lgs_filehdl.h" #include "log/logd/lgs_mbcsv_v1.h" #include "log/logd/lgs_mbcsv_v2.h" -#include "base/osaf_time.h" -#include "osaf/immutil/immutil.h" +#include "log/logd/lgs_oi_admin.h" #define DEFAULT_NUM_APP_LOG_STREAMS 64 @@ -444,15 +446,18 @@ void log_stream_delete(log_stream_t **s) { if (lgs_cb->ha_state == SA_AMF_HA_ACTIVE) { if (stream->isRtStream == SA_TRUE) { - SaAisErrorT rv; + SaAisErrorT ais_rc; TRACE("Stream is closed, I am HA active so remove IMM object"); SaNameT objectName; osaf_extended_name_lend(stream->name.c_str(), &objectName); - rv = saImmOiRtObjectDelete(lgs_cb->immOiHandle, &objectName); - if (rv != SA_AIS_OK) { + ais_rc = saImmOiRtObjectDelete(lgsGetOiHandle(), &objectName); + if (ais_rc != SA_AIS_OK) { + if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + lgsOiCreateBackground(); + } /* no retry; avoid blocking LOG service #1886 */ - LOG_WA("saImmOiRtObjectDelete returned %u for %s", rv, - stream->name.c_str()); + LOG_WA("%s: saImmOiRtObjectDelete returned %s for %s", + __FUNCTION__, saf_error(ais_rc), stream->name.c_str()); } } } @@ -514,6 +519,53 @@ int lgs_populate_log_stream( return rc; } +// Wrap creation of a runtime object. Handle try again ERR_EXIST +// If the object already exist it is deleted and then the new object is created +// Note: This is a help function to be used in createRtObject() only +// Note: immutil is used +// The first 4 parameters are the same as for immutil_saImmOiRtObjectCreate_2 +// The last parameter is the same as object name used with +// immutil_saImmOiRtObjectDelete() +static SaAisErrorT createRtObject(const SaImmOiHandleT oi_handle, + const SaImmClassNameT class_name, + const SaNameT* parent_name, + const SaImmAttrValuesT_2** attribute_values, + const SaNameT *object_name) { +SaAisErrorT ais_rc = SA_AIS_OK; + + for (int i = 0; i <= 2; i++) { + ais_rc = immutil_saImmOiRtObjectCreate_2(oi_handle, + class_name, + parent_name, + attribute_values); + if (ais_rc == SA_AIS_ERR_EXIST) { + SaAisErrorT del_rc = immutil_saImmOiRtObjectDelete(oi_handle, object_name); + if (del_rc == SA_AIS_ERR_BAD_HANDLE) { + LOG_NO("%s: saImmOiRtObjectDelete() Fail, %s", __FUNCTION__, + saf_error(del_rc)); + lgsOiCreateBackground(); + } else if (del_rc != SA_AIS_OK) { + LOG_NO("%s: saImmOiRtObjectDelete() Fail, %s", __FUNCTION__, + saf_error(del_rc)); + break; + } else { + // The existing object is deleted. Try again to create a new one + continue; + } + } else if (ais_rc == SA_AIS_ERR_BAD_HANDLE) { + LOG_NO("%s: saImmOiRtObjectCreate_2() Fail, %s", + __FUNCTION__, saf_error(ais_rc)); + lgsOiCreateBackground(); + break; + } else { + break; + } + } + + // saImmOiRtObjectCreate_2() return code + return ais_rc; +} + /** * Create a new rt app stream object. * @@ -536,8 +588,9 @@ SaAisErrorT lgs_create_appstream_rt_object(log_stream_t *const stream) { parent_name++; /* FIX, vulnerable for malformed DNs */ parentName = &parent; osaf_extended_name_lend(parent_name, &parent); - } else + } else { rdnstr = const_cast<char *>(stream->name.c_str()); + } void *arr1[] = {&rdnstr}; const SaImmAttrValuesT_2 attr_safLgStr = { @@ -622,14 +675,20 @@ SaAisErrorT lgs_create_appstream_rt_object(log_stream_t *const stream) { &attr_saLogStreamCreationTimestamp, NULL}; - rc = immutil_saImmOiRtObjectCreate_2( - lgs_cb->immOiHandle, const_cast<SaImmClassNameT>("SaLogStream"), - parentName, attrValues); + SaNameT object_name; + osaf_extended_name_lend(stream->name.c_str(), &object_name); + SaImmOiHandleT oi_handle = lgsGetOiHandle(); + rc = createRtObject(oi_handle, + const_cast<SaImmClassNameT>("SaLogStream"), + parentName, + attrValues, + &object_name); free(dndup); if (rc != SA_AIS_OK) { - LOG_WA("saImmOiRtObjectCreate_2 returned %u for %s, parent %s", rc, - stream->name.c_str(), parent_name); + LOG_WA("%s: createRtObject() returned %s for %s, " + "parent %s", __FUNCTION__, saf_error(rc), stream->name.c_str(), + parent_name); } } -- 2.16.2 ------------------------------------------------------------------------------ Check out the vibrant tech community on one of the world's most engaging tech sites, Slashdot.org! http://sdm.link/slashdot _______________________________________________ Opensaf-devel mailing list Opensaf-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/opensaf-devel
