Hi Lennart,

Ack (code review only) with few comments inline, tagged [Vu]

Regards, Vu

> -----Original Message-----
> From: Lennart Lund [mailto:lennart.l...@ericsson.com]
> Sent: Tuesday, April 3, 2018 9:59 PM
> To: vu.m.ngu...@dektech.com.au; canh.v.tru...@dektech.com.au
> Cc: opensaf-devel@lists.sourceforge.net; Lennart Lund
> <lennart.l...@ericsson.com>
> Subject: [PATCH 1/1] log: Handling of IMM OI BAD HANDLE in log server is
> incorrect [#2799]
> 
> Recovery of OI handle shall be started in all places where BAD HANDLE
> can be returned. Creation of OI must be done in background thread.
> Ongoing creation must be possible to stop e.g if server is becoming
> standby
> ---
>  src/log/Makefile.am          |   3 +
>  src/log/logd/lgs.h           |  24 ---
>  src/log/logd/lgs_amf.cc      |  25 +--
>  src/log/logd/lgs_cb.h        |   3 -
>  src/log/logd/lgs_config.cc   |  33 ++-
>  src/log/logd/lgs_config.h    |   5 +-
>  src/log/logd/lgs_evt.cc      |  56 +++---
>  src/log/logd/lgs_imm.cc      | 312 +++--------------------------
>  src/log/logd/lgs_imm.h       |  53 +++++
>  src/log/logd/lgs_main.cc     | 116 ++++-------
>  src/log/logd/lgs_mbcsv_v2.cc |   7 +-
>  src/log/logd/lgs_mbcsv_v3.cc |   7 +-
>  src/log/logd/lgs_mbcsv_v5.cc |   3 +-
>  src/log/logd/lgs_oi_admin.cc | 466
> +++++++++++++++++++++++++++++++++++++++++++
>  src/log/logd/lgs_oi_admin.h  | 108 ++++++++++
>  src/log/logd/lgs_recov.cc    |   4 +-
>  src/log/logd/lgs_stream.cc   |  85 ++++++--
>  17 files changed, 848 insertions(+), 462 deletions(-)
>  create mode 100644 src/log/logd/lgs_imm.h
>  create mode 100644 src/log/logd/lgs_oi_admin.cc
>  create mode 100644 src/log/logd/lgs_oi_admin.h
> 
> diff --git a/src/log/Makefile.am b/src/log/Makefile.am
> index 3d951eb5d..5d33d355b 100644
> --- a/src/log/Makefile.am
> +++ b/src/log/Makefile.am
> @@ -79,6 +79,7 @@ noinst_HEADERS += \
>       src/log/logd/lgs_file.h \
>       src/log/logd/lgs_filehdl.h \
>       src/log/logd/lgs_fmt.h \
> +     src/log/logd/lgs_imm.h \
>       src/log/logd/lgs_imm_gcfg.h \
>       src/log/logd/lgs_mbcsv.h \
>       src/log/logd/lgs_mbcsv_v1.h \
> @@ -86,6 +87,7 @@ noinst_HEADERS += \
>       src/log/logd/lgs_mbcsv_v3.h \
>       src/log/logd/lgs_mbcsv_v5.h \
>       src/log/logd/lgs_mbcsv_v6.h \
> +     src/log/logd/lgs_oi_admin.h \
>       src/log/logd/lgs_recov.h \
>       src/log/logd/lgs_stream.h \
>       src/log/logd/lgs_util.h \
> @@ -139,6 +141,7 @@ bin_osaflogd_SOURCES = \
>       src/log/logd/lgs_mbcsv_v5.cc \
>       src/log/logd/lgs_mbcsv_v6.cc \
>       src/log/logd/lgs_mds.cc \
> +     src/log/logd/lgs_oi_admin.cc \
>       src/log/logd/lgs_recov.cc \
>       src/log/logd/lgs_stream.cc \
>       src/log/logd/lgs_util.cc \
> diff --git a/src/log/logd/lgs.h b/src/log/logd/lgs.h
> index 18e6d9281..b1d773375 100644
> --- a/src/log/logd/lgs.h
> +++ b/src/log/logd/lgs.h
> @@ -95,7 +95,6 @@ extern uint32_t mbox_msgs[NCS_IPC_PRIORITY_MAX];
>  extern bool mbox_full[NCS_IPC_PRIORITY_MAX];
>  extern uint32_t mbox_low[NCS_IPC_PRIORITY_MAX];
>  extern pthread_mutex_t lgs_mbox_init_mutex;
> -extern pthread_mutex_t lgs_OI_init_mutex;
> 
>  extern uint32_t initialize_for_assignment(lgs_cb_t *cb, SaAmfHAStateT
> ha_state);
> 
> @@ -108,27 +107,4 @@ extern uint32_t lgs_mds_msg_send(lgs_cb_t *cb,
> lgsv_msg_t *msg, MDS_DEST *dest,
>                                   MDS_SYNC_SND_CTXT *mds_ctxt,
>                                   MDS_SEND_PRIORITY_TYPE prio);
> 
> -extern SaAisErrorT lgs_imm_create_configStream(lgs_cb_t *cb);
> -extern void logRootDirectory_filemove(const std::string
> &new_logRootDirectory,
> -                                      const std::string
&old_logRootDirectory,
> -                                      time_t *cur_time_in);
> -extern void logDataGroupname_fileown(const char
> *new_logDataGroupname);
> -
> -extern void lgs_imm_impl_reinit_nonblocking(lgs_cb_t *cb);
> -extern void lgs_imm_init_OI_handle(SaImmOiHandleT *immOiHandle,
> -                                   SaSelectionObjectT
*immSelectionObject);
> -extern void lgs_imm_impl_set(SaImmOiHandleT *immOiHandle,
> -                             SaSelectionObjectT *immSelectionObject);
> -extern SaAisErrorT lgs_imm_init_configStreams(lgs_cb_t *cb);
> -
> -// Functions for recovery handling
> -void lgs_cleanup_abandoned_streams();
> -void lgs_delete_one_stream_object(const std::string &name_str);
> -void lgs_search_stream_objects();
> -SaUint32T *lgs_get_scAbsenceAllowed_attr(SaUint32T *attr_val);
> -int lgs_get_streamobj_attr(SaImmAttrValuesT_2 ***attrib_out,
> -                           const std::string &object_name,
> -                           SaImmHandleT *immOmHandle);
> -int lgs_free_streamobj_attr(SaImmHandleT immHandle);
> -
>  #endif  // LOG_LOGD_LGS_H_
> diff --git a/src/log/logd/lgs_amf.cc b/src/log/logd/lgs_amf.cc
> index 6fa044ff2..999989146 100644
> --- a/src/log/logd/lgs_amf.cc
> +++ b/src/log/logd/lgs_amf.cc
> @@ -23,6 +23,7 @@
>  #include "osaf/immutil/immutil.h"
>  #include "log/logd/lgs.h"
>  #include "log/logd/lgs_config.h"
> +#include "log/logd/lgs_oi_admin.h"
> 
>  static void close_all_files() {
>    log_stream_t *stream;
> @@ -63,8 +64,7 @@ static SaAisErrorT amf_active_state_handler(lgs_cb_t
> *cb,
>      goto done;
>    }
> 
> -  lgs_imm_impl_set(&cb->immOiHandle, &cb->immSelectionObject);
> -  conf_runtime_obj_create(cb->immOiHandle);
> +  lgsOiStart();
[Vu] In legacy code, only implementer set is done here when switch-over to
make switch-over as fast as possible.
But now, it needs to more jobs: initialize OI handle, set implementer name,
set class implementer and getting selection object.
Is this impact on switchover? Note that, at the time of switch-over - IMM
could be in high work-load.

>    lgs_start_gcfg_applier();
> 
>    // Iterate all existing log streams in cluster.
> @@ -127,12 +127,8 @@ static SaAisErrorT
> amf_quiescing_state_handler(lgs_cb_t *cb,
>    TRACE_ENTER2("HA QUIESCING request");
>    close_all_files();
> 
> -  /* Give up our IMM OI implementer role */
> -  SaAisErrorT ais_rc = immutil_saImmOiImplementerClear(cb-
> >immOiHandle);
> -  if (ais_rc != SA_AIS_OK) {
> -    LOG_WA("immutil_saImmOiImplementerClear failed: %s",
> saf_error(ais_rc));
> -  }
> -
> +  // Give up our IMM OI implementer role and the OpensafConfig class
> applier
> +  lgsOiStop();
>    lgs_stop_gcfg_applier();
> 
>    return saAmfCSIQuiescingComplete(cb->amf_hdl, invocation, SA_AIS_OK);
> @@ -158,11 +154,7 @@ static SaAisErrorT
> amf_quiesced_state_handler(lgs_cb_t *cb,
>    close_all_files();
> 
>    /* Give up our IMM OI implementer role */
> -  SaAisErrorT rc = immutil_saImmOiImplementerClear(cb->immOiHandle);
> -  if (rc != SA_AIS_OK) {
> -    LOG_WA("immutil_saImmOiImplementerClear failed: %s", saf_error(rc));
> -  }
> -
> +  lgsOiStop();
>    lgs_stop_gcfg_applier();
> 
>    /*
> @@ -173,9 +165,10 @@ static SaAisErrorT
> amf_quiesced_state_handler(lgs_cb_t *cb,
> 
>    mds_role = cb->mds_role;
>    cb->mds_role = V_DEST_RL_QUIESCED;
> +  SaAisErrorT ais_rc = SA_AIS_OK;
>    if (lgs_mds_change_role(cb) != NCSCC_RC_SUCCESS) {
>      LOG_ER("lgs_mds_change_role FAILED");
> -    rc = SA_AIS_ERR_FAILED_OPERATION;
> +    ais_rc = SA_AIS_ERR_FAILED_OPERATION;
>      cb->mds_role = mds_role;
>      goto done;
>    }
> @@ -183,8 +176,9 @@ static SaAisErrorT
> amf_quiesced_state_handler(lgs_cb_t *cb,
>    cb->amf_invocation_id = invocation;
>    cb->is_quiesced_set = true;
>  done:
> -  return rc;
> +  return ais_rc;
>  }
> +
> 
> /**************************************************************
> **************
>   * Name          : amf_health_chk_callback
>   *
> @@ -207,6 +201,7 @@ static void amf_health_chk_callback(SaInvocationT
> invocation,
>                                      SaAmfHealthcheckKeyT *checkType) {
>    saAmfResponse(lgs_cb->amf_hdl, invocation, SA_AIS_OK);
>  }
> +
> 
> /**************************************************************
> **************
>   * Name          : amf_csi_set_callback
>   *
> diff --git a/src/log/logd/lgs_cb.h b/src/log/logd/lgs_cb.h
> index f2602c2d8..1b1a67d9c 100644
> --- a/src/log/logd/lgs_cb.h
> +++ b/src/log/logd/lgs_cb.h
> @@ -81,9 +81,6 @@ typedef struct lgs_cb {
>    SaInvocationT
>        amf_invocation_id; /* AMF InvocationID - needed to handle Quiesed
> state */
>    bool is_quiesced_set;
> -  SaImmOiHandleT immOiHandle; /* IMM OI handle
*/
> -  SaSelectionObjectT
> -      immSelectionObject; /* Selection Object to wait for IMM events */
>    SaSelectionObjectT
>        clmSelectionObject;  /* Selection Object to wait for clms events */
>    SaClmHandleT clm_hdl;    /* CLM handle, obtained through CLM init
*/
> diff --git a/src/log/logd/lgs_config.cc b/src/log/logd/lgs_config.cc
> index 4190e3048..60804e835 100644
> --- a/src/log/logd/lgs_config.cc
> +++ b/src/log/logd/lgs_config.cc
> @@ -41,6 +41,7 @@
>  #include "log/logd/lgs_file.h"
>  #include "log/logd/lgs.h"
>  #include "log/logd/lgs_common.h"
> +#include "log/logd/lgs_oi_admin.h"
> 
> 
>  /* Mutex for making read and write of configuration data thread safe */
> @@ -1338,8 +1339,8 @@ static void read_log_config_environ_var_2() {
>   * Public functions for handling configuration information
> 
> **************************************************************
> ****************/
> 
> -void lgs_cfg_init(SaImmOiHandleT immOiHandle, SaAmfHAStateT ha_state) {
> -  TRACE_ENTER2("immOiHandle = %lld", immOiHandle);
> +void lgs_cfg_init() {
> +  TRACE_ENTER();
> 
>    /* Read configuration step 1
>     * Read all values from the log service configuration object
> @@ -1585,6 +1586,10 @@ void conf_runtime_obj_create(SaImmOiHandleT
> immOiHandle) {
> 
>    if (rc == SA_AIS_ERR_EXIST) {
>      TRACE("Server runtime configuration object already exist");
> +  } else if (rc == SA_AIS_ERR_BAD_HANDLE) {
> +    LOG_NO("%s: saImmOiRtObjectCreate_2 BAD_HANDLE, start re-create",
> +           __FUNCTION__);
> +    lgsOiCreateBackground();
>    } else if (rc != SA_AIS_OK) {
>      LOG_NO("%s: Cannot create config runtime object %s", __FUNCTION__,
>             saf_error(rc));
> @@ -1667,6 +1672,14 @@ static SaAisErrorT update_runtime_attrValues(
>    attrMod.modAttr.attrValueType = attrValueType;
>    attrMod.modAttr.attrValues = values_array;
>    ais_rc = immutil_saImmOiRtObjectUpdate_2(immOiHandle, &objectName,
> attrMods);
> +  if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +    LOG_NO("%s: saImmOiRtObjectUpdate_2() Fail, %s", __FUNCTION__,
> +           saf_error(ais_rc));
> +    lgsOiCreateBackground();
> +  } else if (ais_rc != SA_AIS_OK) {
> +    LOG_NO("%s: saImmOiRtObjectUpdate_2() Fail, %s", __FUNCTION__,
> +           saf_error(ais_rc));
> +  }
>    free(values_array);
>    TRACE_LEAVE();
>    return ais_rc;
> @@ -1692,8 +1705,7 @@ static SaAisErrorT
> update_lgs_cfg_runtime_multivalue(
>        immOiHandle, LGS_CFG_RUNTIME_OBJECT, attributeName, valueType,
>        attrValuesNumber, attrValues);
>    if (ais_rc != SA_AIS_OK) {
> -    LOG_NO("%s: update_runtime_attrValues Fail %s", __FUNCTION__,
> -           saf_error(ais_rc));
> +    LOG_NO("%s: update_runtime_attrValues Fail", __FUNCTION__);
>    }
> 
>    // Free the memory allocated by vector_of_strings_to_attrValues()
> @@ -1712,7 +1724,7 @@ static SaAisErrorT
> update_lgs_cfg_runtime_multivalue(
>   * @param immOiHandle[in]
>   * @param attributeNames[in]
>   */
> -void conf_runtime_obj_hdl(SaImmOiHandleT immOiHandle,
> +void conf_runtime_obj_handler(SaImmOiHandleT immOiHandle,
>                            const SaImmAttrNameT *attributeNames) {
>    SaImmAttrNameT attributeName;
>    int i = 0;
> @@ -1799,9 +1811,14 @@ void conf_runtime_obj_hdl(SaImmOiHandleT
> immOiHandle,
>        TRACE("%s: unknown attribute %s", __FUNCTION__, attributeName);
>      }
> 
> -    if (ais_rc != SA_AIS_OK) {
> -      LOG_ER("immutil_update_one_rattr (%s) failed: %s", attributeName,
> -             saf_error(ais_rc));
> +    if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +      LOG_WA("%s: immutil_update_one_rattr (%s) failed: %s",
> __FUNCTION__,
> +             attributeName, saf_error(ais_rc));
> +      lgsOiCreateBackground();
> +      break;
> +    } else if (ais_rc != SA_AIS_OK) {
> +      LOG_ER("%s: immutil_update_one_rattr (%s) failed: %s",
> __FUNCTION__,
> +             attributeName, saf_error(ais_rc));
>        osaf_abort(0);
>      }
>    }
> diff --git a/src/log/logd/lgs_config.h b/src/log/logd/lgs_config.h
> index 6885271a4..3f1b05e51 100644
> --- a/src/log/logd/lgs_config.h
> +++ b/src/log/logd/lgs_config.h
> @@ -135,7 +135,7 @@ typedef struct config_chkpt {
>   * Read the log service configuration data verify and update
configuration
>   * data structure
>   */
> -void lgs_cfg_init(SaImmOiHandleT immOiHandle, SaAmfHAStateT ha_state);
> +void lgs_cfg_init();
> 
>  /**
>   * Get value of log service configuration parameter from the
configuration
> data
> @@ -316,8 +316,9 @@ void lgs_groupnameconf_set(const char
> *data_groupname_str);
>   * Handle runtime object for showing actual configuration and
configuration
>   * related information
>   */
> +
>  void conf_runtime_obj_create(SaImmOiHandleT immOiHandle);
> -void conf_runtime_obj_hdl(SaImmOiHandleT immOiHandle,
> +void conf_runtime_obj_handler(SaImmOiHandleT immOiHandle,
>                            const SaImmAttrNameT *attributeNames);
> 
>  /*
> diff --git a/src/log/logd/lgs_evt.cc b/src/log/logd/lgs_evt.cc
> index 4b735875d..589cb262b 100644
> --- a/src/log/logd/lgs_evt.cc
> +++ b/src/log/logd/lgs_evt.cc
> @@ -31,6 +31,7 @@
>  #include "log/logd/lgs_imm_gcfg.h"
>  #include "log/logd/lgs_clm.h"
>  #include "log/logd/lgs_dest.h"
> +#include "log/logd/lgs_oi_admin.h"
> 
>  void *client_db = nullptr; /* used for C++ STL map */
> 
> @@ -569,7 +570,7 @@ static uint32_t proc_rda_cb_msg(lgsv_lgs_evt_t
> *evt) {
>      }
> 
>      /* fail over, become implementer */
> -    lgs_imm_impl_set(&lgs_cb->immOiHandle, &lgs_cb-
> >immSelectionObject);
> +    lgsOiCreateBackground();
>      lgs_start_gcfg_applier();
> 
>      /* Agent down list has to be processed first */
> @@ -613,7 +614,6 @@ uint32_t lgs_cb_init(lgs_cb_t *lgs_cb) {
> 
>    lgs_cb->fully_initialized = false;
>    lgs_cb->amfSelectionObject = -1;
> -  lgs_cb->immSelectionObject = -1;
>    lgs_cb->mbcsv_sel_obj = -1;
>    lgs_cb->clm_hdl = 0;
>    lgs_cb->clmSelectionObject = -1;
> @@ -835,7 +835,7 @@ snd_rsp:
>   */
>  SaAisErrorT create_new_app_stream(lgsv_stream_open_req_t
> *open_sync_param,
>                                    log_stream_t **o_stream) {
> -  SaAisErrorT rc = SA_AIS_OK;
> +  SaAisErrorT ais_rc = SA_AIS_OK;
>    log_stream_t *stream;
>    SaBoolT twelveHourModeFlag;
>    SaUint32T logMaxLogrecsize_conf = 0;
> @@ -848,13 +848,13 @@ SaAisErrorT
> create_new_app_stream(lgsv_stream_open_req_t *open_sync_param,
> 
>    if (lgs_is_extended_name_valid(&open_sync_param->lstr_name) == false) {
>      TRACE("SaNameT is invalid");
> -    rc = SA_AIS_ERR_INVALID_PARAM;
> +    ais_rc = SA_AIS_ERR_INVALID_PARAM;
>      goto done;
>    }
> 
>    if (open_sync_param->logFileFullAction !=
> SA_LOG_FILE_FULL_ACTION_ROTATE) {
>      TRACE("Unsupported logFileFullAction");
> -    rc = SA_AIS_ERR_NOT_SUPPORTED;
> +    ais_rc = SA_AIS_ERR_NOT_SUPPORTED;
>      goto done;
>    }
> 
> @@ -864,7 +864,7 @@ SaAisErrorT
> create_new_app_stream(lgsv_stream_open_req_t *open_sync_param,
>        ((open_sync_param->maxFilesRotated < 1) ||
>         (open_sync_param->maxFilesRotated > 127))) {
>      TRACE("Invalid maxFilesRotated. Valid Range = [1-127]");
> -    rc = SA_AIS_ERR_INVALID_PARAM;
> +    ais_rc = SA_AIS_ERR_INVALID_PARAM;
>      goto done;
>    }
> 
> @@ -880,21 +880,21 @@ SaAisErrorT
> create_new_app_stream(lgsv_stream_open_req_t *open_sync_param,
>                                        STREAM_TYPE_APPLICATION_RT,
>                                        &twelveHourModeFlag)) {
>      TRACE("format expression failure");
> -    rc = SA_AIS_ERR_INVALID_PARAM;
> +    ais_rc = SA_AIS_ERR_INVALID_PARAM;
>      goto done;
>    }
> 
>    /* Verify if there is any special character in logFileName */
>    if (lgs_has_special_char(open_sync_param->logFileName) == true) {
>      TRACE("Invalid logFileName - %s", open_sync_param->logFileName);
> -    rc = SA_AIS_ERR_INVALID_PARAM;
> +    ais_rc = SA_AIS_ERR_INVALID_PARAM;
>      goto done;
>    }
> 
>    /* Verify if logFileName length is valid */
>    if (lgs_is_valid_filelength(open_sync_param->logFileName) == false) {
>      TRACE("logFileName is invalid");
> -    rc = SA_AIS_ERR_INVALID_PARAM;
> +    ais_rc = SA_AIS_ERR_INVALID_PARAM;
>      goto done;
>    }
> 
> @@ -902,7 +902,7 @@ SaAisErrorT
> create_new_app_stream(lgsv_stream_open_req_t *open_sync_param,
>    if (lgs_is_valid_pathlength(open_sync_param->logFilePathName,
>                                open_sync_param->logFileName) == false) {
>      TRACE("logFilePathName is invalid");
> -    rc = SA_AIS_ERR_INVALID_PARAM;
> +    ais_rc = SA_AIS_ERR_INVALID_PARAM;
>      goto done;
>    }
> 
> @@ -913,7 +913,7 @@ SaAisErrorT
> create_new_app_stream(lgsv_stream_open_req_t *open_sync_param,
>      if ((stream->fileName == open_sync_param->logFileName) &&
>          (stream->pathName == open_sync_param->logFilePathName)) {
>        TRACE("pathname already exist");
> -      rc = SA_AIS_ERR_INVALID_PARAM;
> +      ais_rc = SA_AIS_ERR_INVALID_PARAM;
>        goto done;
>      }
>    }
> @@ -922,7 +922,7 @@ SaAisErrorT
> create_new_app_stream(lgsv_stream_open_req_t *open_sync_param,
>    str_name = osaf_extended_name_borrow(&open_sync_param-
> >lstr_name);
>    if (strncmp(dnPrefix, str_name, strlen(dnPrefix)) != 0) {
>      TRACE("'%s' is not a valid stream name => invalid param", str_name);
> -    rc = SA_AIS_ERR_INVALID_PARAM;
> +    ais_rc = SA_AIS_ERR_INVALID_PARAM;
>      goto done;
>    }
> 
> @@ -933,13 +933,13 @@ SaAisErrorT
> create_new_app_stream(lgsv_stream_open_req_t *open_sync_param,
>        ((open_sync_param->maxLogRecordSize < SA_LOG_MIN_RECORD_SIZE)
> ||
>         (open_sync_param->maxLogRecordSize > logMaxLogrecsize_conf))) {
>      TRACE("maxLogRecordSize is invalid");
> -    rc = SA_AIS_ERR_INVALID_PARAM;
> +    ais_rc = SA_AIS_ERR_INVALID_PARAM;
>      goto done;
>    }
> 
>    *o_stream = log_stream_new(str_name, STREAM_NEW);
>    if (*o_stream == nullptr) {
> -    rc = SA_AIS_ERR_NO_MEMORY;
> +    ais_rc = SA_AIS_ERR_NO_MEMORY;
>      goto done;
>    }
> 
> @@ -951,16 +951,19 @@ SaAisErrorT
> create_new_app_stream(lgsv_stream_open_req_t *open_sync_param,
>        twelveHourModeFlag, 0, *o_stream);  // output
>    if (err == -1) {
>      log_stream_delete(o_stream);
> -    rc = SA_AIS_ERR_NO_MEMORY;
> +    ais_rc = SA_AIS_ERR_NO_MEMORY;
>      goto done;
>    }
> 
> -  rc = lgs_create_appstream_rt_object(*o_stream);
> -  if (rc != SA_AIS_OK) log_stream_delete(o_stream);
> +  ais_rc = lgs_create_appstream_rt_object(*o_stream);
> +  if (ais_rc != SA_AIS_OK) log_stream_delete(o_stream);
> +
> +  TRACE("%s: lgs_create_appstream_rt_object return, %s",
> +         __FUNCTION__, saf_error(ais_rc));
> 
>  done:
>    TRACE_LEAVE();
> -  return rc;
> +  return ais_rc;
>  }
> 
>  /**
> @@ -1069,15 +1072,7 @@ static uint32_t proc_stream_open_msg(lgs_cb_t
> *cb, lgsv_lgs_evt_t *evt) {
>        }
>      }
>    } else {
> -    /* Stream does not exist */
> -
> -    // This check is to avoid the client getting SA_AIS_BAD_OPERATION
> -    // as there is no IMM OI implementer set.
> -    if (cb->immOiHandle == 0) {
> -      TRACE("IMM service unavailable, open stream failed");
> -      ais_rv = SA_AIS_ERR_TRY_AGAIN;
> -      goto snd_rsp;
> -    }
> +    /* Stream does not exist. Create a new stream */
> 
>      /*
>       * Check if the stream is in the list of stream objects
> @@ -1125,6 +1120,11 @@ static uint32_t proc_stream_open_msg(lgs_cb_t
> *cb, lgsv_lgs_evt_t *evt) {
>      if (ais_rv != SA_AIS_OK) {
>        TRACE("%s create_new_app_stream Fail \"%s\"", __FUNCTION__,
>              saf_error(ais_rv));
> +      if (ais_rv == SA_AIS_ERR_BAD_HANDLE) {
> +        // This means that the stream RT object could not be created
because
> +        // of a bad OI handle. Change to TRY AGAIN in the reply to the
agent
> +        ais_rv = SA_AIS_ERR_TRY_AGAIN;
> +      }
>        goto snd_rsp;
>      }
>    }
> @@ -1218,7 +1218,7 @@ static uint32_t proc_stream_close_msg(lgs_cb_t
> *cb, lgsv_lgs_evt_t *evt) {
>    // This check is to avoid the client getting SA_AIS_BAD_OPERATION
>    // as there is no IMM OI implementer set.
>    if ((stream->streamType == STREAM_TYPE_APPLICATION_RT) &&
> -      (cb->immOiHandle == 0)) {
> +      (lgsGetOiHandle() == 0)) {
>      TRACE("IMM service unavailable, close stream failed");
>      ais_rc = SA_AIS_ERR_TRY_AGAIN;
>      goto snd_rsp;
> diff --git a/src/log/logd/lgs_imm.cc b/src/log/logd/lgs_imm.cc
> index 60870fa17..d23862172 100644
> --- a/src/log/logd/lgs_imm.cc
> +++ b/src/log/logd/lgs_imm.cc
> @@ -31,6 +31,8 @@
>  #define _GNU_SOURCE
>  #endif
> 
> +#include "log/logd/lgs_imm.h"
> +
>  #include <stdio.h>
>  #include <errno.h>
>  #include <string.h>
> @@ -46,6 +48,7 @@
>  #include "log/logd/lgs_recov.h"
>  #include "log/logd/lgs_config.h"
>  #include "log/logd/lgs_dest.h"
> +#include "log/logd/lgs_oi_admin.h"
>  #include "base/saf_error.h"
> 
>  #include "lgs_mbcsv_v1.h"
> @@ -58,11 +61,6 @@
>   * ----------------
>   */
> 
> -/* Used for protecting global imm OI handle and selection object during
> - * initialize of OI
> - */
> -pthread_mutex_t lgs_OI_init_mutex = PTHREAD_MUTEX_INITIALIZER;
> -
>  /* Used for checkpointing time when files are closed */
>  static time_t chkp_file_close_time = 0;
> 
> @@ -75,13 +73,6 @@ static const char *log_file_format[] = {
>      DEFAULT_ALM_NOT_FORMAT_EXP, DEFAULT_ALM_NOT_FORMAT_EXP,
>      DEFAULT_APP_SYS_FORMAT_EXP, DEFAULT_APP_SYS_FORMAT_EXP};
> 
> -static const SaImmOiImplementerNameT implementerName =
> -    const_cast<SaImmOiImplementerNameT>("safLogService");
> -static const SaImmClassNameT logConfig_str =
> -    const_cast<SaImmClassNameT>("OpenSafLogConfig");
> -static const SaImmClassNameT streamConfig_str =
> -    const_cast<SaImmClassNameT>("SaLogStreamConfig");
> -
>  // The list contains path-file names when validating its name in CCB
> completed
>  // callback. This help for log service prevents that creating any streams
>  // in same CCB with duplicated log path-file name
> @@ -2375,7 +2366,7 @@ static SaAisErrorT stream_create_and_configure1(
> 
>    /* Update creation timestamp */
>    rc = immutil_update_one_rattr(
> -      lgs_cb->immOiHandle, objectName.c_str(),
> +      lgsGetOiHandle(), objectName.c_str(),
>        const_cast<SaImmAttrNameT>("saLogStreamCreationTimestamp"),
>        SA_IMM_ATTR_SATIMET, &(*stream)->creationTimeStamp);
>    if (rc != SA_AIS_OK) {
> @@ -2702,7 +2693,7 @@ static SaAisErrorT
> rtAttrUpdateCallback(SaImmOiHandleT immOiHandle,
>    /* Handle configuration runtime object */
>    if (strncmp(objName, LGS_CFG_RUNTIME_OBJECT, strlen(objName)) == 0) {
>      /* Handle Runtome configuration object */
> -    conf_runtime_obj_hdl(immOiHandle, attributeNames);
> +    conf_runtime_obj_handler(immOiHandle, attributeNames);
>    } else {
>      /* Handle stream object if valid
>       */
> @@ -2889,11 +2880,15 @@ static const SaImmOiCallbacksT_2 callbacks = {
>      .saImmOiCcbObjectModifyCallback = ccbObjectModifyCallback,
>      .saImmOiRtAttrUpdateCallback = rtAttrUpdateCallback};
> 
> +// Export the callback structure. OI init is done in lgs_oi_admin
> +const SaImmOiCallbacksT_2* getImmOiCallbacks(void) {
> +  return &callbacks;
> +}
> +
>  /**
>   * Retrieve the LOG stream configuration from IMM using the
>   * IMM-OM interface and initialize the corresponding information
> - * in the LOG control block. Initialize the LOG IMM-OI
> - * interface. Become class implementer.
> + * in the LOG control block.
>   *
>   * @param cb[in] control block
>   * @return
> @@ -2987,20 +2982,6 @@ SaAisErrorT lgs_imm_init_configStreams(lgs_cb_t
> *cb) {
>      }
>    }
> 
> -  /* 1.Become implementer
> -   * 2.Update creation timestamp for all configure object, must be object
> -   * implementer first 3.Open all streams Config file and log file will
be
> -   * created. If this fails we give up without returning any error. A new
> -   * attempt to create the files will be done when trying to write a log
> -   * record to the stream.
> -   */
> -  ais_rc =
> -      immutil_saImmOiClassImplementerSet(cb->immOiHandle,
> "SaLogStreamConfig");
> -  if (ais_rc != SA_AIS_OK) {
> -    LOG_ER("immutil_saImmOiClassImplementerSet failed %s",
> saf_error(ais_rc));
> -    osaf_abort(0);
> -  }
> -
>    // Iterate all existing log streams in cluster.
>    while ((stream = iterate_all_streams(endloop, jstart)) && !endloop) {
>      jstart = SA_FALSE;
> @@ -3021,7 +3002,7 @@ SaAisErrorT lgs_imm_init_configStreams(lgs_cb_t
> *cb) {
>      }
> 
>      ais_rc = immutil_update_one_rattr(
> -        cb->immOiHandle, stream->name.c_str(),
> +        lgsGetOiHandle(), stream->name.c_str(),
>          const_cast<SaImmAttrNameT>("saLogStreamCreationTimestamp"),
>          SA_IMM_ATTR_SATIMET, &stream->creationTimeStamp);
>      if (ais_rc != SA_AIS_OK) {
> @@ -3046,255 +3027,6 @@ done:
>    return ais_rc;
>  }
> 
> -/**
> - * Initialize the OI interface and get a selection object.
> - * Become OI for safLogService if Active
> - *
> - * @param immOiHandle[out]
> - * @param immSelectionObject[out]
> - */
> -void lgs_imm_init_OI_handle(SaImmOiHandleT *immOiHandle,
> -                            SaSelectionObjectT *immSelectionObject) {
> -  SaAisErrorT rc;
> -  uint32_t msecs_waited = 0;
> -
> -  TRACE_ENTER();
> -
> -  /* Initialize IMM OI service */
> -  SaVersionT imm_version = kImmVersion;
> -  rc = saImmOiInitialize_2(immOiHandle, &callbacks, &imm_version);
> -  while ((rc == SA_AIS_ERR_TRY_AGAIN) &&
> -         (msecs_waited < max_waiting_time_60s)) {
> -    usleep(sleep_delay_ms * 1000);
> -    msecs_waited += sleep_delay_ms;
> -    imm_version = kImmVersion;
> -    rc = saImmOiInitialize_2(immOiHandle, &callbacks, &imm_version);
> -  }
> -  if (rc != SA_AIS_OK) {
> -    lgs_exit("saImmOiInitialize_2 failed", SA_AMF_COMPONENT_RESTART);
> -  }
> -
> -  /* Get selection object for event handling */
> -  msecs_waited = 0;
> -  rc = saImmOiSelectionObjectGet(*immOiHandle, immSelectionObject);
> -  while ((rc == SA_AIS_ERR_TRY_AGAIN) &&
> -         (msecs_waited < max_waiting_time_10s)) {
> -    usleep(sleep_delay_ms * 1000);
> -    msecs_waited += sleep_delay_ms;
> -    rc = saImmOiSelectionObjectGet(*immOiHandle, immSelectionObject);
> -  }
> -  if (rc != SA_AIS_OK) {
> -    lgs_exit("saImmOiSelectionObjectGet failed",
> SA_AMF_COMPONENT_RESTART);
> -  }
> -
> -  TRACE_LEAVE2("rc: %s", saf_error(rc));
> -}
> -
> -/**
> - * Does the sequence of setting an implementer name and class implementer
> - *
> - * @param immOiHandle[in]
> - * @return SaAisErrorT
> - */
> -static SaAisErrorT imm_impl_set_sequence(
> -    SaImmOiHandleT *immOiHandle, SaSelectionObjectT
> *immSelectionObject) {
> -  SaAisErrorT rc = SA_AIS_OK;
> -  uint32_t msecs_waited = 0;
> -
> -  TRACE_ENTER();
> -
> -  for (;;) {
> -    if (msecs_waited >= max_waiting_time_60s) {
> -      TRACE("Timeout in imm_impl_set_sequence");
> -      goto done;
> -    }
> -
> -    /* Become object implementer
> -     */
> -    rc = saImmOiImplementerSet(*immOiHandle, implementerName);
> -    while (((rc == SA_AIS_ERR_TRY_AGAIN) || (rc == SA_AIS_ERR_EXIST)) &&
> -           (msecs_waited < max_waiting_time_60s)) {
> -      usleep(sleep_delay_ms * 1000);
> -      msecs_waited += sleep_delay_ms;
> -      rc = saImmOiImplementerSet(*immOiHandle, implementerName);
> -    }
> -    if (rc == SA_AIS_ERR_BAD_HANDLE || rc == SA_AIS_ERR_TIMEOUT) {
> -      LOG_WA("saImmOiImplementerSet returned %s", saf_error(rc));
> -      usleep(sleep_delay_ms * 1000);
> -      msecs_waited += sleep_delay_ms;
> -      saImmOiFinalize(*immOiHandle);
> -      *immOiHandle = 0;
> -      *immSelectionObject = -1;
> -      lgs_imm_init_OI_handle(immOiHandle, immSelectionObject);
> -      continue;
> -    }
> -    if (rc != SA_AIS_OK) {
> -      TRACE("saImmOiImplementerSet failed %s", saf_error(rc));
> -      goto done;
> -    }
> -
> -    /*
> -     * Become class implementer for the OpenSafLogConfig class if it
exists
> -     * Become class implementer for the SaLogStreamConfig class
> -     */
> -    if (true == *static_cast<const bool *>(
> -
> lgs_cfg_get(LGS_IMM_LOG_OPENSAFLOGCONFIG_CLASS_EXIST))) {
> -      rc = saImmOiClassImplementerSet(*immOiHandle, logConfig_str);
> -      while (((rc == SA_AIS_ERR_TRY_AGAIN) || (rc == SA_AIS_ERR_EXIST))
&&
> -             (msecs_waited < max_waiting_time_60s)) {
> -        usleep(sleep_delay_ms * 1000);
> -        msecs_waited += sleep_delay_ms;
> -        rc = saImmOiClassImplementerSet(*immOiHandle, logConfig_str);
> -      }
> -      if (rc == SA_AIS_ERR_BAD_HANDLE || rc == SA_AIS_ERR_TIMEOUT) {
> -        LOG_WA("saImmOiClassImplementerSet returned %s", saf_error(rc));
> -        usleep(sleep_delay_ms * 1000);
> -        msecs_waited += sleep_delay_ms;
> -        saImmOiFinalize(*immOiHandle);
> -        *immOiHandle = 0;
> -        *immSelectionObject = -1;
> -        lgs_imm_init_OI_handle(immOiHandle, immSelectionObject);
> -        continue;
> -      }
> -      if (rc != SA_AIS_OK) {
> -        TRACE("saImmOiClassImplementerSet OpenSafLogConfig failed %s",
> -              saf_error(rc));
> -        goto done;
> -      }
> -    }
> -
> -    rc = saImmOiClassImplementerSet(*immOiHandle, streamConfig_str);
> -    while (((rc == SA_AIS_ERR_TRY_AGAIN) || (rc == SA_AIS_ERR_EXIST)) &&
> -           (msecs_waited < max_waiting_time_60s)) {
> -      usleep(sleep_delay_ms * 1000);
> -      msecs_waited += sleep_delay_ms;
> -      rc = saImmOiClassImplementerSet(*immOiHandle, streamConfig_str);
> -    }
> -    if (rc == SA_AIS_ERR_BAD_HANDLE || rc == SA_AIS_ERR_TIMEOUT) {
> -      LOG_WA("saImmOiClassImplementerSet returned %s", saf_error(rc));
> -      usleep(sleep_delay_ms * 1000);
> -      msecs_waited += sleep_delay_ms;
> -      saImmOiFinalize(*immOiHandle);
> -      *immOiHandle = 0;
> -      *immSelectionObject = -1;
> -      lgs_imm_init_OI_handle(immOiHandle, immSelectionObject);
> -      continue;
> -    }
> -    if (rc != SA_AIS_OK) {
> -      TRACE("saImmOiClassImplementerSet SaLogStreamConfig failed %s",
> -            saf_error(rc));
> -      goto done;
> -    }
> -    break;
> -  }
> -
> -done:
> -  TRACE_LEAVE2("rc: %s", saf_error(rc));
> -  return rc;
> -}
> -
> -/**
> - * Set implementer name and become class implementer.
> - * This function will block until done.
> - *
> - * @param cb
> - */
> -void lgs_imm_impl_set(SaImmOiHandleT *immOiHandle,
> -                      SaSelectionObjectT *immSelectionObject) {
> -  SaAisErrorT rc = SA_AIS_OK;
> -
> -  TRACE_ENTER();
> -
> -  rc = imm_impl_set_sequence(immOiHandle, immSelectionObject);
> -  if (rc != SA_AIS_OK) {
> -    lgs_exit("Becoming OI implementer failed",
> SA_AMF_COMPONENT_RESTART);
> -  }
> -
> -  TRACE_LEAVE();
> -}
> -
> -/**
> - * Thread
> - * Restore object and class implementer/applier.
> - *
> - * @param _cb[in]
> - */
> -static void *imm_impl_init_thread(void *_cb) {
> -  lgs_cb_t *cb = static_cast<lgs_cb_t *>(_cb);
> -  SaSelectionObjectT immSelectionObject = 0;
> -  SaImmOiHandleT immOiHandle = 0;
> -  SaAisErrorT rc = SA_AIS_OK;
> -
> -  TRACE_ENTER();
> -
> -  /* Initialize handles and become implementer */
> -  lgs_imm_init_OI_handle(&immOiHandle, &immSelectionObject);
> -  rc = imm_impl_set_sequence(&immOiHandle, &immSelectionObject);
> -  if (rc != SA_AIS_OK) {
> -    lgs_exit("Becoming OI implementer failed",
> SA_AMF_COMPONENT_RESTART);
> -  }
> -
> -  /* Store handle and selection object.
> -   * Protect if the poll loop in main is released during the storage
> -   * sequence.
> -   */
> -  osaf_mutex_lock_ordie(&lgs_OI_init_mutex);
> -  cb->immSelectionObject = immSelectionObject;
> -  cb->immOiHandle = immOiHandle;
> -  osaf_mutex_unlock_ordie(&lgs_OI_init_mutex);
> -
> -  /* Activate the poll loop in main()
> -   * This will reinstall IMM poll event handling
> -   */
> -  lgsv_lgs_evt_t *lgsv_evt;
> -  lgsv_evt = static_cast<lgsv_lgs_evt_t *>(calloc(1,
sizeof(lgsv_lgs_evt_t)));
> -  osafassert(lgsv_evt);
> -  lgsv_evt->evt_type = LGSV_EVT_NO_OP;
> -  if (m_NCS_IPC_SEND(&lgs_mbx, lgsv_evt, LGS_IPC_PRIO_CTRL_MSGS) !=
> -      NCSCC_RC_SUCCESS) {
> -    LOG_WA("imm_reinit_thread failed to send IPC message to main
thread");
> -    /*
> -     * Se no reason why this would happen. But if it does at least there
> -     * is something in the syslog. The main thread should still pick up
> -     * the new imm FD when there is a healthcheck, but it could take
> -     *minutes.
> -     */
> -    free(lgsv_evt);
> -  }
> -
> -  TRACE_LEAVE();
> -  return NULL;
> -}
> -
> -/**
> - * In a separate thread:
> - * Initiate IMM OI handle and selection object
> - * Create imm implementer for log IMM objects
> - * When complete:
> - * Store the new handle and selection object in the cb store.
> - * Activate the poll loop in main() by sending an empty mailbox message
> - *
> - * @param cb[out]
> - */
> -void lgs_imm_impl_reinit_nonblocking(lgs_cb_t *cb) {
> -  pthread_t thread;
> -  pthread_attr_t attr;
> -
> -  TRACE_ENTER();
> -
> -  pthread_attr_init(&attr);
> -  pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
> -
> -  if (pthread_create(&thread, &attr, imm_impl_init_thread, cb) != 0) {
> -    LOG_ER("pthread_create FAILED: %s", strerror(errno));
> -    exit(EXIT_FAILURE);
> -  }
> -
> -  pthread_attr_destroy(&attr);
> -
> -  TRACE_LEAVE();
> -}
> -
> 
> /**************************************************************
> ****************
>   * Functions used for recovery handling
> 
> **************************************************************
> ****************/
> @@ -3403,12 +3135,16 @@ void lgs_delete_one_stream_object(const
> std::string &name_str) {
>    osaf_extended_name_lend(name_str.c_str(), &object_name);
> 
>    /* and delete the object */
> -  ais_rc = immutil_saImmOiRtObjectDelete(lgs_cb->immOiHandle,
> &object_name);
> +  ais_rc = immutil_saImmOiRtObjectDelete(lgsGetOiHandle(),
> &object_name);
>    if (ais_rc == SA_AIS_OK) {
>      TRACE("%s Object \"%s\" deleted", __FUNCTION__, name_str.c_str());
> +  } else if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +    LOG_NO("%s: saImmOiRtObjectDelete() Fail, %s", __FUNCTION__,
> +           saf_error(ais_rc));
> +    lgsOiCreateBackground();
>    } else {
> -    LOG_WA("%s saImmOiRtObjectDelete for \"%s\" FAILED %d",
> __FUNCTION__,
> -           name_str.c_str(), ais_rc);
> +    LOG_WA("%s saImmOiRtObjectDelete for \"%s\" FAILED %s",
> __FUNCTION__,
> +           name_str.c_str(), saf_error(ais_rc));
>    }
>  }
> 
> @@ -3444,12 +3180,16 @@ void lgs_cleanup_abandoned_streams() {
>        /* Copy name to a SaNameT */
>        osaf_extended_name_lend(name_str, &object_name);
>        /* and delete the object */
> -      ais_rc = immutil_saImmOiRtObjectDelete(lgs_cb->immOiHandle,
> &object_name);
> +      ais_rc = immutil_saImmOiRtObjectDelete(lgsGetOiHandle(),
> &object_name);
>        if (ais_rc == SA_AIS_OK) {
>          TRACE("\tObject \"%s\" deleted", name_str);
> +      } else if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +        LOG_WA("%s saImmOiRtObjectDelete for \"%s\" FAILED %s",
> __FUNCTION__,
> +               name_str, saf_error(ais_rc));
> +        lgsOiCreateBackground();
>        } else {
> -        LOG_WA("%s saImmOiRtObjectDelete for \"%s\" FAILED %d",
> __FUNCTION__,
> -               name_str, ais_rc);
> +        LOG_WA("%s saImmOiRtObjectDelete for \"%s\" FAILED %s",
> __FUNCTION__,
> +               name_str, saf_error(ais_rc));
>        }
>      } else {
>        /* Should never happen! */
> diff --git a/src/log/logd/lgs_imm.h b/src/log/logd/lgs_imm.h
> new file mode 100644
> index 000000000..195d09380
> --- /dev/null
> +++ b/src/log/logd/lgs_imm.h
> @@ -0,0 +1,53 @@
> +/*      -*- OpenSAF  -*-
> + *
> + * (C) Copyright 2008 The OpenSAF Foundation
> + * Copyright Ericsson AB 2008, 2018 - All Rights Reserved.
> + *
> + * This program is distributed in the hope that it will be useful, but
> + * WITHOUT ANY WARRANTY; without even the implied warranty of
> MERCHANTABILITY
> + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are
> licensed
> + * under the GNU Lesser General Public License Version 2.1, February
1999.
> + * The complete license can be accessed from the following location:
> + * http://opensource.org/licenses/lgpl-license.php
> + * See the Copying file included with the OpenSAF distribution for full
> + * licensing terms.
> + *
> + * Author(s): Ericsson AB
> + *
> + */
> +
> +#ifndef LGS_IMM_H
> +#define LGS_IMM_H
> +
> +#include <time.h>
> +#include <string>
> +
> +#include "ais/include/saImmOi.h"
> +#include "ais/include/saImmOm.h"
> +
> +#include "log/logd/lgs_cb.h"
> +
> +// Get the callback structure used when initializing an OI handle
> +const SaImmOiCallbacksT_2* getImmOiCallbacks(void);
> +
> +void logRootDirectory_filemove(const std::string &new_logRootDirectory,
> +                                      const std::string
&old_logRootDirectory,
> +                                      time_t *cur_time_in);
> +void logDataGroupname_fileown(const char *new_logDataGroupname);
> +
> +SaAisErrorT lgs_imm_init_configStreams(lgs_cb_t *cb);
> +
> +// Functions for recovery handling
> +void lgs_cleanup_abandoned_streams();
> +void lgs_delete_one_stream_object(const std::string &name_str);
> +void lgs_search_stream_objects();
> +SaUint32T *lgs_get_scAbsenceAllowed_attr(SaUint32T *attr_val);
> +int lgs_get_streamobj_attr(SaImmAttrValuesT_2 ***attrib_out,
> +                           const std::string &object_name,
> +                           SaImmHandleT *immOmHandle);
> +int lgs_free_streamobj_attr(SaImmHandleT immHandle);
> +
> +
> +
> +#endif /* LGS_IMM_H */
> +
> diff --git a/src/log/logd/lgs_main.cc b/src/log/logd/lgs_main.cc
> index 988ebdd78..9767fe00d 100644
> --- a/src/log/logd/lgs_main.cc
> +++ b/src/log/logd/lgs_main.cc
> @@ -33,16 +33,18 @@
>  #include "nid/agent/nid_api.h"
>  #include "base/ncs_main_papi.h"
>  #include "base/osaf_time.h"
> -
> -#include "log/logd/lgs.h"
> -#include "log/logd/lgs_file.h"
>  #include "base/osaf_utility.h"
>  #include "base/hash.h"
> -#include "lgs_recov.h"
>  #include "osaf/immutil/immutil.h"
> +
> +#include "log/logd/lgs.h"
> +#include "log/logd/lgs_file.h"
> +#include "log/logd/lgs_recov.h"
>  #include "lgs_clm.h"
>  #include "log/logd/lgs_dest.h"
>  #include "log/logd/lgs_amf.h"
> +#include "log/logd/lgs_oi_admin.h"
> +#include "log/logd/lgs_imm.h"
> 
> 
>  /*
> ==============================================================
> ==========
> @@ -383,26 +385,21 @@ uint32_t initialize_for_assignment(lgs_cb_t *cb,
> SaAmfHAStateT ha_state) {
>    if (cb->fully_initialized || ha_state == SA_AMF_HA_QUIESCED) goto done;
>    cb->ha_state = ha_state;
> 
> -  /* Initialize IMM OI handle and selection object */
> -  lgs_imm_init_OI_handle(&cb->immOiHandle, &cb->immSelectionObject);
> -
> -  TRACE("IMM init done: cb->immOiHandle = %lld", cb->immOiHandle);
> -
>    /* Initialize log configuration
>     * Must be done after IMM OI is initialized
>     */
> -  lgs_cfg_init(cb->immOiHandle, cb->ha_state);
> +  lgs_cfg_init();
>    lgs_trace_config(); /* Show all configuration in TRACE */
> 
> -  /* Show some configurtion info in sysylog */
> -  logsv_root_dir =
> -      static_cast<const char
> *>(lgs_cfg_get(LGS_IMM_LOG_ROOT_DIRECTORY));
> -  logsv_data_groupname =
> -      static_cast<const char *>(lgs_cfg_get(LGS_IMM_DATA_GROUPNAME));
> +  /* Show some configuration info in sysylog */
> +  logsv_root_dir = static_cast<const char *>
> +                   (lgs_cfg_get(LGS_IMM_LOG_ROOT_DIRECTORY));
> +  logsv_data_groupname = static_cast<const char *>
> +                         (lgs_cfg_get(LGS_IMM_DATA_GROUPNAME));
>    LOG_NO("LOG root directory is: \"%s\"", logsv_root_dir);
>    LOG_NO("LOG data group is: \"%s\"", logsv_data_groupname);
> -  vdest = reinterpret_cast<const std::vector<std::string> *>(
> -      lgs_cfg_get(LGS_IMM_LOG_RECORD_DESTINATION_CONFIGURATION));
> +  vdest = reinterpret_cast<const std::vector<std::string> *>
> +
> (lgs_cfg_get(LGS_IMM_LOG_RECORD_DESTINATION_CONFIGURATION));
>    osafassert(vdest != nullptr);
>    if (vdest->size() > 0) {
>      CfgDestination(*vdest, ModifyType::kAdd);
> @@ -445,20 +442,15 @@ uint32_t initialize_for_assignment(lgs_cb_t *cb,
> SaAmfHAStateT ha_state) {
>    }
> 
>    if (ha_state == SA_AMF_HA_ACTIVE) {
> -    /* Become OI. We will be blocked here until done */
> -    lgs_imm_impl_set(&cb->immOiHandle, &cb->immSelectionObject);
> -    conf_runtime_obj_create(cb->immOiHandle);
> +    // Request an Object Implementer, a configuration object handler and
> +    // an applier for opensafNetworkName change surveillance
> +    lgsOiCreateSynchronous();
> +    conf_runtime_obj_create(lgsGetOiHandle());
>      lgs_start_gcfg_applier();
> 
> -    /* Create streams that has configuration objects and become
> -     * class implementer for the SaLogStreamConfig class
> -     */
> -
> -    /* Note1: cb->immOiHandle is set in lgs_imm_init()
> -     * Note2: cb->logsv_root_dir must be set
> -     */
> +    // Create streams that has configuration objects
>      if (lgs_imm_init_configStreams(cb) != SA_AIS_OK) {
> -      LOG_ER("lgs_imm_create_configStream FAILED");
> +      LOG_ER("lgs_imm_create_configStreams FAILED");
>        rc = NCSCC_RC_FAILURE;
>        goto done;
>      }
> @@ -479,7 +471,7 @@ done:
>   */
>  int main(int argc, char *argv[]) {
>    NCS_SEL_OBJ mbx_fd;
> -  SaAisErrorT error = SA_AIS_OK;
> +  SaAisErrorT ais_rc = SA_AIS_OK;
>    uint32_t rc;
>    int term_fd;
> 
> @@ -519,7 +511,7 @@ int main(int argc, char *argv[]) {
>    fds[FD_AMF].events = POLLIN;
>    fds[FD_MBX].fd = mbx_fd.rmv_obj;
>    fds[FD_MBX].events = POLLIN;
> -  fds[FD_IMM].fd = lgs_cb->immSelectionObject;
> +  fds[FD_IMM].fd = lgsGetOiSelectionObject();
>    fds[FD_IMM].events = POLLIN;
> 
>    lgs_cb->clmSelectionObject = lgs_cb->clm_init_sel_obj.rmv_obj;
> @@ -539,18 +531,16 @@ int main(int argc, char *argv[]) {
>      fds[FD_CLM].fd = lgs_cb->clmSelectionObject;
>      fds[FD_CLM].events = POLLIN;
> 
> -    /* Protect since the reinit thread may be in the process of
> -     * changing the values
> -     */
> -    osaf_mutex_lock_ordie(&lgs_OI_init_mutex);
> -    if (lgs_cb->immOiHandle != 0) {
> -      fds[FD_IMM].fd = lgs_cb->immSelectionObject;
> +    // Adds or removes the IMM fd depending on if there is an OI or not
> +    SaImmOiHandleT oi_handle;
> +    SaSelectionObjectT oi_selection_object =
> lgsGetOiFdsParams(&oi_handle);
> +    if (oi_handle != 0) {
> +      fds[FD_IMM].fd = oi_selection_object;
>        fds[FD_IMM].events = POLLIN;
>        nfds = FD_IMM + 1;
>      } else {
>        nfds = FD_IMM;
>      }
> -    osaf_mutex_unlock_ordie(&lgs_OI_init_mutex);
> 
>      int ret = poll(fds, nfds, -1);
> 
> @@ -567,9 +557,9 @@ int main(int argc, char *argv[]) {
> 
>      if (fds[FD_AMF].revents & POLLIN) {
>        if (lgs_cb->amf_hdl != 0) {
> -        if ((error = saAmfDispatch(lgs_cb->amf_hdl, SA_DISPATCH_ALL)) !=
> +        if ((ais_rc = saAmfDispatch(lgs_cb->amf_hdl, SA_DISPATCH_ALL)) !=
>              SA_AIS_OK) {
> -          LOG_ER("saAmfDispatch failed: %u", error);
> +          LOG_ER("saAmfDispatch failed: %u", ais_rc);
>            break;
>          }
>        } else {
> @@ -593,9 +583,9 @@ int main(int argc, char *argv[]) {
> 
>      if (fds[FD_CLM].revents & POLLIN) {
>        if (lgs_cb->clm_hdl != 0) {
> -        if ((error = saClmDispatch(lgs_cb->clm_hdl, SA_DISPATCH_ALL)) !=
> +        if ((ais_rc = saClmDispatch(lgs_cb->clm_hdl, SA_DISPATCH_ALL)) !=
>              SA_AIS_OK) {
> -          LOG_ER("saClmDispatch failed: %u", error);
> +          LOG_ER("saClmDispatch failed: %u", ais_rc);
>            break;
>          }
>        } else {
> @@ -629,40 +619,16 @@ int main(int argc, char *argv[]) {
> 
>      if (fds[FD_MBX].revents & POLLIN) lgs_process_mbx(&lgs_mbx);
> 
> -    if (lgs_cb->immOiHandle && fds[FD_IMM].revents & POLLIN) {
> -      error = saImmOiDispatch(lgs_cb->immOiHandle, SA_DISPATCH_ALL);
> -
> -      /*
> -       * BAD_HANDLE is interpreted as an IMM service restart. Try
> -       * reinitialize the IMM OI API in a background thread and let
> -       * this thread do business as usual especially handling write
> -       * requests.
> -       *
> -       * All other errors are treated as non-recoverable (fatal) and will
> -       * cause an exit of the process.
> -       */
> -      if (error == SA_AIS_ERR_BAD_HANDLE) {
> -        TRACE("saImmOiDispatch returned BAD_HANDLE");
> -
> -        /*
> -         * Invalidate the IMM OI handle, this info is used in other
> -         * locations. E.g. giving TRY_AGAIN responses to a create and
> -         * close app stream requests. That is needed since the IMM OI
> -         * is used in context of these functions.
> -         *
> -         * Also closing the handle. Finalize is ok with a bad handle
> -         * that is bad because it is stale and this actually clears
> -         * the handle from internal agent structures.  In any case
> -         * we ignore the return value from Finalize here.
> -         */
> -        saImmOiFinalize(lgs_cb->immOiHandle);
> -        lgs_cb->immOiHandle = 0;
> -        lgs_cb->immSelectionObject = -1;
> -
> -        /* Initiate IMM reinitializtion in the background */
> -        lgs_imm_impl_reinit_nonblocking(lgs_cb);
> -      } else if (error != SA_AIS_OK) {
> -        LOG_ER("saImmOiDispatch FAILED: %u", error);
> +    if (fds[FD_IMM].revents & POLLIN) {
> +      ais_rc = saImmOiDispatch(lgsGetOiHandle(), SA_DISPATCH_ALL);
> +
> +      if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +        LOG_NO("saImmOiDispatch returned BAD_HANDLE");
> +
> +        // Request creation of a new OI
> +        lgsOiCreateBackground();
> +      } else if (ais_rc != SA_AIS_OK) {
> +        LOG_ER("saImmOiDispatch FAILED: %u", ais_rc);
>          break;
>        }
>      }
> diff --git a/src/log/logd/lgs_mbcsv_v2.cc b/src/log/logd/lgs_mbcsv_v2.cc
> index 9afc30155..63807e1b7 100644
> --- a/src/log/logd/lgs_mbcsv_v2.cc
> +++ b/src/log/logd/lgs_mbcsv_v2.cc
> @@ -21,9 +21,10 @@
>   * version 2 check-pointing. See also lgs_mbcsv_v2.h
>   */
> 
> -#include "lgs_config.h"
> -#include "lgs_mbcsv_v2.h"
> -#include "lgs_mbcsv.h"
> +#include "log/logd/lgs_config.h"
> +#include "log/logd/lgs_imm.h"
> +#include "log/logd/lgs_mbcsv_v2.h"
> +#include "log/logd/lgs_mbcsv.h"
> 
> 
> /**************************************************************
> **************
>   * Name          : ckpt_proc_lgs_cfg
> diff --git a/src/log/logd/lgs_mbcsv_v3.cc b/src/log/logd/lgs_mbcsv_v3.cc
> index f9836c8af..75fd98dba 100644
> --- a/src/log/logd/lgs_mbcsv_v3.cc
> +++ b/src/log/logd/lgs_mbcsv_v3.cc
> @@ -21,9 +21,10 @@
>   * version 3 check-pointing. See also lgs_mbcsv_v3.h
>   */
> 
> -#include "lgs_mbcsv_v3.h"
> -#include "lgs_mbcsv_v2.h"
> -#include "lgs_mbcsv.h"
> +#include "log/logd/lgs_imm.h"
> +#include "log/logd/lgs_mbcsv_v3.h"
> +#include "log/logd/lgs_mbcsv_v2.h"
> +#include "log/logd/lgs_mbcsv.h"
> 
> 
> /**************************************************************
> **************
>   * Name          : ckpt_proc_lgs_cfg
> diff --git a/src/log/logd/lgs_mbcsv_v5.cc b/src/log/logd/lgs_mbcsv_v5.cc
> index 75d627688..d219f8a5b 100644
> --- a/src/log/logd/lgs_mbcsv_v5.cc
> +++ b/src/log/logd/lgs_mbcsv_v5.cc
> @@ -22,8 +22,9 @@
>   * V5 applies to log server configuration check-pointing
>   */
> 
> -#include "lgs_mbcsv_v5.h"
>  #include "log/logd/lgs_dest.h"
> +#include "log/logd/lgs_imm.h"
> +#include "log/logd/lgs_mbcsv_v5.h"
> 
> 
> /**************************************************************
> **************
>   * Name          : ckpt_proc_lgs_cfg
> diff --git a/src/log/logd/lgs_oi_admin.cc b/src/log/logd/lgs_oi_admin.cc
> new file mode 100644
> index 000000000..ce1fc0ad3
> --- /dev/null
> +++ b/src/log/logd/lgs_oi_admin.cc
> @@ -0,0 +1,466 @@
> +/*      -*- OpenSAF  -*-
> + *
> + * (C) Copyright 2008 The OpenSAF Foundation
> + * Copyright Ericsson AB 2008, 2018 - All Rights Reserved.
> + *
> + * This program is distributed in the hope that it will be useful, but
> + * WITHOUT ANY WARRANTY; without even the implied warranty of
> MERCHANTABILITY
> + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are
> licensed
> + * under the GNU Lesser General Public License Version 2.1, February
1999.
> + * The complete license can be accessed from the following location:
> + * http://opensource.org/licenses/lgpl-license.php
> + * See the Copying file included with the OpenSAF distribution for full
> + * licensing terms.
> + *
> + * Author(s): Ericsson AB
> + *
> + */
[Vu] This is the new file. Copyright should be changed.

> +
> +#include "log/logd/lgs_oi_admin.h"
> +
> +#include <atomic>
> +#include <thread>
> +
> +#include "ais/include/saAis.h"
> +#include "ais/include/saImmOi.h"
> +#include "base/logtrace.h"
> +#include "base/mutex.h"
> +#include "base/saf_error.h"
> +#include "base/time.h"
> +
> +#include "log/logd/lgs.h"
> +#include "log/logd/lgs_imm.h"
> +#include "log/logd/lgs_config.h"
> +#include "log/logd/lgs_util.h"
> +
> +// Note: Always use the set and get function with these varibales
> +static base::Mutex lgs_OI_init_mutex;
> +static SaImmOiHandleT oi_handle_ = 0;
> +static SaSelectionObjectT oi_selection_object_ = -1;
> +
> +static const SaImmOiImplementerNameT kImplementerName =
> +    const_cast<SaImmOiImplementerNameT>("safLogService");
> +// The log server is OI for the following configuration classes
> +static const SaImmClassNameT kLogConfigurationClass =
> +    const_cast<SaImmClassNameT>("OpenSafLogConfig");
> +static const SaImmClassNameT kLogStreamClass =
> +    const_cast<SaImmClassNameT>("SaLogStreamConfig");
> +
> +// Max time for creation of OI
> +static const uint64_t kOiCreationTimeout = 90000;  // 1.5 minutes
> +// Max time for getting an OI handle
> +static const uint64_t kOiHandleRequestTimeout = 60000;  // 1 minute
> +// Max time for other APIs in the OI creation sequence
> +static const uint64_t kOiRequestTimeout = 10000;  // 10 sec
> +// Max time to wait for the thread to stop if OI create stop is requested
> +static const uint64_t kOiWaitStopTimeout = 11000; //  11 sec
> +// Delay time used in timed out wait loops
> +static const timespec kOiTryAgainDelay = base::kOneHundredMilliseconds;
> +
> +// Internal help functions
> +// ---------------------------------
> +
> +static void setProtectedGlobals(SaImmOiHandleT oi_handle,
> +                                SaSelectionObjectT oi_selction_object) {
> +  TRACE_ENTER();
> +  base::Lock protect_globals(lgs_OI_init_mutex);
> +  oi_handle_ = oi_handle;
> +  oi_selection_object_ = oi_selction_object;
> +}
> +
> +static SaAisErrorT finalizeOi(SaImmOiHandleT oi_handle) {
> +  TRACE_ENTER();
> +  SaAisErrorT ais_rc = SA_AIS_OK;
> +  base::Timer try_again_timer(kOiRequestTimeout);
> +  while (try_again_timer.is_timeout() == false) {
> +    ais_rc = saImmOiFinalize(oi_handle);
> +    if (ais_rc == SA_AIS_ERR_TRY_AGAIN) {
> +      base::Sleep(kOiTryAgainDelay);
> +      continue;
> +    } else if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +      // If we get BAD HANDLE the OI is already finalized
> +      ais_rc = SA_AIS_OK;
> +      break;
> +    } else if (ais_rc != SA_AIS_OK) {
> +      TRACE("%s: saImmOiFinalize() Fail, %s", __FUNCTION__,
> saf_error(ais_rc));
> +    }
> +    break;
> +  }
> +  if ((try_again_timer.is_timeout() == true) &&
> +      (ais_rc == SA_AIS_ERR_TRY_AGAIN)) {
> +    LOG_NO("%s: saImmOiFinalize() Fail, TRY AGAIN timeout",
> __FUNCTION__);
> +  }
> +
> +  return ais_rc;
> +}
> +
> +// IMM API help function used in the OI creation thread
> +// Communication flag read in OI thread and set by the lgs_OI_delete()
and
> +// create functions. When this flag is true the thread shall stop and
return
> +// as soon as possible
> +// Note1: All of these functions will stop internal loops and return if
> +//        the global atomic flag oi_stop == true
> +// Note2: Only the function used for stopping the OI is allowed to write
this
> +//        flag
> +static std::atomic<bool> stop_oi_create(false);
> +// Flag indicating whether the OI creation thread is running or not
> +static std::atomic<bool> oi_create_is_executing(false);
> +
> +// Output: out_oi_handle. Set to 0 if fail
> +static SaAisErrorT initOiHandle(SaImmOiHandleT *out_oi_handle) {
> +  TRACE_ENTER();
> +  SaAisErrorT ais_rc = SA_AIS_OK;
> +  const SaImmOiCallbacksT_2 *callbacks = getImmOiCallbacks();
> +  base::Timer try_again_timer(kOiHandleRequestTimeout);
> +  while (try_again_timer.is_timeout() == false) {
> +    if (stop_oi_create) {
> +      ais_rc = SA_AIS_OK;
> +      break;
> +    }
> +    SaVersionT imm_version = kImmVersion;
> +    ais_rc = saImmOiInitialize_2(out_oi_handle, callbacks, &imm_version);
> +    if (ais_rc == SA_AIS_ERR_TRY_AGAIN) {
> +      base::Sleep(kOiTryAgainDelay);
> +      continue;
> +    }
> +    if (ais_rc != SA_AIS_OK) {
> +      LOG_WA("%s: saImmOiInitialize_2() Fail, %s", __FUNCTION__,
> +            saf_error(ais_rc));
> +      *out_oi_handle = 0;
> +    }
> +    break;
> +  }
> +  if ((try_again_timer.is_timeout() == true) &&
> +      (ais_rc == SA_AIS_ERR_TRY_AGAIN)) {
> +    LOG_WA("%s: saImmOiInitialize_2() Fail, TRY AGAIN timeout",
> __FUNCTION__);
> +    *out_oi_handle = 0;
> +  }
> +
> +  return ais_rc;
> +}
> +
> +// Return -1 if Fail
> +// out_ais_rc is set to return code from saImmOiSelectionObjectGet()
> +static SaSelectionObjectT requestOiSelectionObject(SaImmOiHandleT
> oi_handle,
> +                                                   SaAisErrorT*
out_ais_rc) {
> +  TRACE_ENTER();
> +  SaSelectionObjectT oi_selection_object = -1;
> +  SaAisErrorT ais_rc = SA_AIS_OK;
> +  base::Timer try_again_timer(kOiRequestTimeout);
> +  while (try_again_timer.is_timeout() == false) {
> +    if (stop_oi_create) {
> +      oi_selection_object = -1;
> +      break;
> +    }
> +    ais_rc = saImmOiSelectionObjectGet(oi_handle, &oi_selection_object);
> +    if (ais_rc == SA_AIS_ERR_TRY_AGAIN) {
> +      base::Sleep(kOiTryAgainDelay);
> +      continue;
> +    }
> +    if (ais_rc != SA_AIS_OK) {
> +      LOG_WA("%s: saImmOiSelectionObjectGet() Fail, %s", __FUNCTION__,
> +             saf_error(ais_rc));
> +      oi_selection_object = -1;
> +    }
> +    break;
> +  }
> +  if ((try_again_timer.is_timeout() == true) &&
> +      (ais_rc == SA_AIS_ERR_TRY_AGAIN)) {
> +    LOG_WA("%s: saImmOiSelectionObjectGet() Fail, TRY AGAIN timeout",
> +           __FUNCTION__);
> +    oi_selection_object = -1;
> +  }
> +
> +  *out_ais_rc = ais_rc;
> +  return oi_selection_object;
> +}
> +
> +// Set implementer name in kImplementerName
> +static SaAisErrorT setOiImplementerName(SaImmOiHandleT oi_handle) {
> +  TRACE_ENTER();
> +  SaAisErrorT ais_rc = SA_AIS_OK;
> +  base::Timer try_again_timer(kOiRequestTimeout);
> +  while (try_again_timer.is_timeout() == false) {
> +    if (stop_oi_create) {
> +      ais_rc = SA_AIS_OK;
> +      break;
> +    }
> +    ais_rc = saImmOiImplementerSet(oi_handle, kImplementerName);
> +    if (ais_rc == SA_AIS_ERR_TRY_AGAIN) {
> +      base::Sleep(kOiTryAgainDelay);
> +      continue;
> +    } else if (ais_rc == SA_AIS_ERR_EXIST) {
> +      // Should be released. May be temporary. Lets try again
> +      TRACE("%s saImmOiImplementerSet() Fail %s. Try to create again",
> +             __FUNCTION__, saf_error(ais_rc));
> +      base::Sleep(kOiTryAgainDelay);
> +      continue;
> +    } else if (ais_rc != SA_AIS_OK) {
> +      LOG_WA("%s: saImmOiImplementerSet() Fail, %s", __FUNCTION__,
> +             saf_error(ais_rc));
> +      break;
> +    }
> +    break;
> +  }
> +  if ((try_again_timer.is_timeout() == true) &&
> +      (ais_rc == SA_AIS_ERR_TRY_AGAIN)) {
> +    LOG_WA("%s: saImmOiImplementerSet() Fail, TRY AGAIN timeout",
> +           __FUNCTION__);
> +  }
> +
> +  TRACE_LEAVE2("ais_rc = %s", saf_error(ais_rc));
> +  return ais_rc;
> +}
> +
> +static SaAisErrorT setClassImplementer(SaImmOiHandleT oi_handle,
> +                                       const SaImmClassNameT class_name)
{
> +  TRACE_ENTER();
> +  SaAisErrorT ais_rc = SA_AIS_OK;
> +  base::Timer try_again_timer(kOiRequestTimeout);
> +  while (try_again_timer.is_timeout() == false) {
> +    if (stop_oi_create) {
> +      ais_rc = SA_AIS_OK;
> +      break;
> +    }
> +    ais_rc = saImmOiClassImplementerSet(oi_handle, class_name);
> +    if (ais_rc == SA_AIS_ERR_TRY_AGAIN) {
> +      base::Sleep(kOiTryAgainDelay);
> +      continue;
> +    }
> +    if (ais_rc != SA_AIS_OK) {
> +      LOG_WA("%s: saImmOiClassImplementerSet() Fail, %s", __FUNCTION__,
> +             saf_error(ais_rc));
> +    }
> +    break;
> +  }
> +  if ((try_again_timer.is_timeout() == true) &&
> +      (ais_rc == SA_AIS_ERR_TRY_AGAIN)) {
> +    LOG_WA("%s: saImmOiClassImplementerSet() Fail, TRY AGAIN timeout",
> +           __FUNCTION__);
> +  }
> +
> +  return ais_rc;
> +}
> +
> +// Activate the poll loop in main()
> +// This will reinstall IMM poll event handling
> +static void sendOiCreatedMessageToMainThread() {
> +  TRACE_ENTER();
> +  // Note: lgsv_evt must be allocated C style since it will be freed in
other
> +  //       part of the code with free()
> +  lgsv_lgs_evt_t *lgsv_evt;
> +  lgsv_evt = static_cast<lgsv_lgs_evt_t *>(calloc(1,
sizeof(lgsv_lgs_evt_t)));
> +  osafassert(lgsv_evt);
> +
> +  lgsv_evt->evt_type = LGSV_EVT_NO_OP;
> +  if (m_NCS_IPC_SEND(&lgs_mbx, lgsv_evt, LGS_IPC_PRIO_CTRL_MSGS) !=
> +      NCSCC_RC_SUCCESS) {
> +    LOG_WA("imm_reinit_thread failed to send IPC message to main
> thread");
> +    /*
> +     * Se no reason why this would happen. But if it does at least there
> +     * is something in the syslog. The main thread should still pick up
> +     * the new imm FD when there is a healthcheck, but it could take
> +     *minutes.
> +     */
> +    free(lgsv_evt);
> +  }
> +}
> +
> +// Creates the log service OI
> +// stop_oi_create:  Used to stop thread execution as soon as possible
> +//                  Will stop the thread when set to true.
> +static void createLogServerOi() {
> +  TRACE_ENTER();
> +  SaAisErrorT ais_rc = SA_AIS_OK;
> +  SaImmOiHandleT oi_handle = 0;
> +  SaSelectionObjectT oi_selection_object = -1;
> +  base::Timer io_create_timeout(kOiCreationTimeout);
> +
> +  while (io_create_timeout.is_timeout() == false) {
> +    // OI init sequence
> +    // Start by finalize OI (handle) in case of sequence recovery
> +    // See continue below
> +    finalizeOi(oi_handle);
> +
> +    // Init OI handle
> +    if (stop_oi_create) break;
> +    ais_rc = initOiHandle(&oi_handle);
> +    if (ais_rc != SA_AIS_OK) {
> +      LOG_NO("%s: initOiHandle() Fail", __FUNCTION__);
> +      oi_handle = 0;
> +      break;
> +    }
> +
> +    // Set OI implementer name
> +    if (stop_oi_create) break;
> +    ais_rc = setOiImplementerName(oi_handle);
> +    if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +      TRACE("%s: setOiImplementerName() Fail %s. Try to recover",
> +            __FUNCTION__, saf_error(ais_rc));
> +      continue;
> +    } else if (ais_rc != SA_AIS_OK) {
> +      LOG_NO("%s: setOiImplementerName() Fail", __FUNCTION__);
> +      break;
> +    }
> +
> +    // Become class implementer for log config class
> +    if (stop_oi_create) break;
> +    ais_rc = setClassImplementer(oi_handle, kLogConfigurationClass);
> +    if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +      TRACE("%s: setClassImplementer(LogConfigurationClass) Fail %s. "
> +            "Try to recover", __FUNCTION__, saf_error(ais_rc));
> +      continue;
> +    } else if (ais_rc != SA_AIS_OK) {
> +      LOG_NO("%s: setClassImplementer(LogConfigurationClass) Fail",
> +             __FUNCTION__);
> +      break;
> +    }
> +
> +    // Become class implementer for stream config class
> +    if (stop_oi_create) break;
> +    ais_rc = setClassImplementer(oi_handle, kLogStreamClass);
> +    if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +      TRACE("%s: setClassImplementer(kLogStreamClass) Fail %s. "
> +            "Try to recover", __FUNCTION__, saf_error(ais_rc));
> +      continue;
> +    } else if (ais_rc != SA_AIS_OK) {
> +      LOG_NO("%s: setClassImplementer(kLogStreamClass) Fail",
> __FUNCTION__);
> +      break;
> +    } else {
> +      // Get the selection object
> +      oi_selection_object = requestOiSelectionObject(oi_handle, &ais_rc);
> +      if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +        TRACE("%s: requestOiSelectionObject() Fail", __FUNCTION__);
> +      }
> +      break;
> +    }
> +  }
> +
> +  // Creation is done, has failed or is stopped.
> +  if (stop_oi_create) {
> +    TRACE("%s: OI create is stopped", __FUNCTION__);
> +    ais_rc = SA_AIS_ERR_NO_OP;  // Dummy. Must not be SA_AIS_OK
[Vu] I think, we should check if the local OI handle is valid or not. If
valid, do finalize the handle, otherwise it will be leaked and OI
implementer will not be cleared.
> +  } else if ((io_create_timeout.is_timeout() == true) &&
> +             (ais_rc != SA_AIS_OK)) {
[Vu] Is this checking useless? Only below "ais_rc != SA_AIS_OK" is enough?
> +    LOG_WA("%s: Fail, OI creation timeout", __FUNCTION__);
> +    // The legacy behavior is to exit log service in this case
> +    lgs_exit("createLogServerOi() failed", SA_AMF_COMPONENT_RESTART);
> +  } else if (ais_rc != SA_AIS_OK) {
> +    LOG_WA("%s: Fail, OI creation error", __FUNCTION__);
> +    // The legacy behavior is to exit log service in this case
> +    lgs_exit("createLogServerOi() failed", SA_AMF_COMPONENT_RESTART);
> +  }
> +
> +  if (ais_rc == SA_AIS_OK) {
> +    // Save OI handle and selection object
> +    // Note: In case of stop_oi_create this is handled in the stop
function
> +    setProtectedGlobals(oi_handle, oi_selection_object);
> +  }
[Vu] Can above logics be replaced by below?

if (ais_rc != SA_AIS_OK && stop_oi_create == false) {
        lgs_exit("createLogServerOi() failed", SA_AMF_COMPONENT_RESTART);
}
setProtectedGlobals(oi_handle, oi_selection_object);


> +}
> +
> +// The thread function for creating a log server OI
> +// stop_oi_create:  Used to stop thread execution as soon as possible
> +//                  Will stop the thread when set to true.
> +static void oiCreationThread() {
> +  TRACE_ENTER();
> +  createLogServerOi();
> +  sendOiCreatedMessageToMainThread();
> +  oi_create_is_executing = false;
> +  return;
> +}
> +
> +// Public functions
> +//-----------------
> +
> +SaImmOiHandleT lgsGetOiHandle() {
> +  base::Lock protect_globals(lgs_OI_init_mutex);
> +  return oi_handle_;
> +}
> +
> +SaSelectionObjectT lgsGetOiSelectionObject() {
> +  base::Lock protect_globals(lgs_OI_init_mutex);
> +  return oi_selection_object_;
> +}
> +
> +SaSelectionObjectT lgsGetOiFdsParams(SaImmOiHandleT* out_oi_handle) {
> +  base::Lock protect_globals(lgs_OI_init_mutex);
> +  *out_oi_handle = oi_handle_;
> +  return oi_selection_object_;
> +}
> +
> +void lgsOiCreateBackground() {
> +  TRACE_ENTER();
> +  if (oi_create_is_executing == true) {
> +    // Thread is executing. Do nothing
> +    return;
> +  }
> +
> +  // Make sure that the OI is finalized before creating a new one
> +  finalizeOi(oi_handle_);
> +  setProtectedGlobals(0, -1);
> +
> +  // Start a OI creation thread
> +  // Note: Throws an exception if "The system is unable to start a new
> thread"
> +  // Should never happen so it is not handled here (will cause node
restart)
> +  oi_create_is_executing = true;
> +  std::thread (&oiCreationThread).detach();
> +}
> +
> +void lgsOiCreateSynchronous() {
> +  TRACE_ENTER();
> +  // Note: oi_handle_ can be used unprotected here since we know that
> there is
> +  // no running oiCreationThread
> +  // There is no meaning to call finalizeOI() since this function only
shall be
> +  // used for the first creation of an OI handle during initialization of
the
> +  // log service
> +  oi_handle_ = 0;
> +  oi_selection_object_ = -1;
> +  createLogServerOi();
> +}
> +
> +// Note: In this version this is just call lgsOiCreateBackground(). The
reason
> +// for having this function is to have a place where an implementer set
can
> +// be done instead of creating an OI if the lgsOiStop function is changed
> +// to just clear the OI. Note that if set fail the OI must be created
anyway
> +//
> +// This function shall be used only when changing HA state from standby
to
> +// active
> +void lgsOiStart() {
> +  TRACE_ENTER();
> +  lgsOiCreateBackground();
> +}
> +
> +// Note: oi_handle_ and oi_selection_object_ is always set in thread also
if
> +// thread is stopped. Is set to 0 and -1 respectively in all cases except
when
> +// an OI is successfully created
> +//
> +// Note: This version will completely finalize the OI. This is probably
good
> +//       enough. Clear / Set handling may be a bit faster but Stop /
Start of
> +//       OI should be rare.
> +//       If optimization is needed we can do implementer clear here
> +void lgsOiStop() {
> +  TRACE_ENTER();
> +  // This is a request for the OI creation thread to stop
> +  stop_oi_create = true;
> +
> +  // Wait for thread to stop if it is running
> +  base::Timer io_thread_stop_timeout(kOiWaitStopTimeout);
> +  while (io_thread_stop_timeout.is_timeout() != true) {
> +    // Wait for the OI creation thread to stop (return)
> +    // Note: The thread has to be detached so join cannot be used
> +    if (oi_create_is_executing == false) break;
> +    base::Sleep(base::kOneHundredMilliseconds);
> +  }
> +  if ((io_thread_stop_timeout.is_timeout() == true) &&
> +      (oi_create_is_executing == true)) {
> +    // The OI creation thread is hanging. Recover by restarting log
server
> +    lgs_exit("lgsOiStop: Stopping OI create thread timeout",
> +             SA_AMF_COMPONENT_RESTART);
> +  }
> +
> +  // Note: oi_handle_ can be used unprotected here since we know that
> there is
> +  // no running oiCreationThread
> +  finalizeOi(oi_handle_);
> +  setProtectedGlobals(0, -1);
> +  stop_oi_create = false;
> +}
> diff --git a/src/log/logd/lgs_oi_admin.h b/src/log/logd/lgs_oi_admin.h
> new file mode 100644
> index 000000000..751824271
> --- /dev/null
> +++ b/src/log/logd/lgs_oi_admin.h
> @@ -0,0 +1,108 @@
> +/*      -*- OpenSAF  -*-
> + *
> + * (C) Copyright 2008 The OpenSAF Foundation
> + * Copyright Ericsson AB 2008, 2018 - All Rights Reserved.
> + *
> + * This program is distributed in the hope that it will be useful, but
> + * WITHOUT ANY WARRANTY; without even the implied warranty of
> MERCHANTABILITY
> + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are
> licensed
> + * under the GNU Lesser General Public License Version 2.1, February
1999.
> + * The complete license can be accessed from the following location:
> + * http://opensource.org/licenses/lgpl-license.php
> + * See the Copying file included with the OpenSAF distribution for full
> + * licensing terms.
> + *
> + * Author(s): Ericsson AB
> + *
> + */
> +
> +#include "ais/include/saImmOi.h"
> +
> +// Contains administrative handling of the lgs main object implementer
(OI)
> +// Implements functions for initializing an OI in a background thread,
> +// finalizing the OI and stop an ongoing initialization
> +// The reason for using a background thread is to not block the log
server.
> +// An initialize request may take up to 1 minute.
> +// When stopping the OI a running OI initializing thread is stopped and
the OI
> +// is finalized.
> +//
> +// Examples of situations when the functions are used:
> +// Starting log server: OI shall be intialized. This should be done in
> +//                      background so that start of log server is not
blocked.
> +//
> +// Switch/fail over:    The to become standby; must give up the OI. The
> chosen
> +//                      way of doing that is to finalize the OI. An
alternative
> +//                      is to only clear the OI which is no longer used
here.
> +//                      The to become active; must initialize an OI. An
> +//                      alternative is to keep the OI handle and just use
the
> +//                      set API
> +//
> +// BAD HANDLE:          IMM may return bad handle when synchronous
> operations
> +//                      are called including the dispatch function. If
this
> +//                      happen the OI shall be initialized again except
in a
> +//                      switch/fail over situation where the server shall
become
> +//                      standby
> +//
> +// Note1: The background thread puts a no-operation message in the
> mailbox
> +//        handled by the main poll loop when OI initialize is done. This
is
> +//        needed to handle installing of the IMM fd in the poll loop.
> +//
> +// Note2: The previous global immOiHandle and immSelectionObject
> variables in
> +//        cb struct are removed. Values can be fetched using thread safe
get
> +//        functions
> +
> +#ifndef LGS_OI_ADMIN_H
> +#define LGS_OI_ADMIN_H
> +
> +// Creates a new OI. Return false if fail.
> +// Note1: If there is an existing OI it will be deleted and a new is
created.
> +//        This means calling IMM finalize, setting oi_handle = 0,
> +//        oi_selection_object = -1 and starting the OI creation thread.
The
> +//        OI creation thread will update oi_handle and
oi_selection_object with
> +//        new values when creation is done.
> +//        Nothing will be done if OI creation is already ongoing when
this
> +//        function is called
> +// Note2: lgsOiCreateBackground() shall not be used when log server is
> starting.
> +//        During initialization configuration streams are created.
> +//        When creating a configuration stream cached Rt attributes has
to be
> +//        written and this requires a valid OI handle
> +//        Instead use lgsOiCreateSyncronous()
> +//
> +// This function shall also be used to restore the OI in case of a BAD
HANDLE
> +// return code when using the OI handle with an OI IMM operation.
> +// Not thread safe. Shall be used in the log server main thread only
> +void lgsOiCreateBackground();
> +
> +// Note3: This function shall only be called once and only when the log
> server
> +//        is initializing
> +// This function is not thread safe
> +void lgsOiCreateSynchronous();
> +
> +// Start an OI.
> +// Shall be used when this server shall become an OI.
> +// For example when becoming active
> +// Not thread safe. Shall be used in the log server main thread only
> +void lgsOiStart();
> +
> +// Stop an existing OI
> +// To be used when this server no longer shall be an OI.
> +// For example becoming standby
> +// Not thread safe. Shall be used in the log server main thread only
> +void lgsOiStop();
> +
> +// Get both OI handle and selection object. Guarantees that we get a pair
> +// that is created together. Return -1 if there is no selection object
and
> +// the value of out_oi_handle will be 0
> +// Is thread safe
> +SaSelectionObjectT lgsGetOiFdsParams(SaImmOiHandleT* out_oi_handle);
> +
> +// Returns the created OI selection object or -1 if there is none
> +// Is thread safe
> +SaSelectionObjectT lgsGetOiSelectionObject();
> +
> +// Returns the created OI handle or 0 if there is no handle
> +// Is thread safe
> +SaImmOiHandleT lgsGetOiHandle();
> +
> +#endif /* LGS_OI_ADMIN_H */
> +
> diff --git a/src/log/logd/lgs_recov.cc b/src/log/logd/lgs_recov.cc
> index dcaad188d..7c0197b7f 100644
> --- a/src/log/logd/lgs_recov.cc
> +++ b/src/log/logd/lgs_recov.cc
> @@ -15,9 +15,11 @@
>   *
>   */
> 
> -#include "lgs_recov.h"
> +#include "log/logd/lgs_recov.h"
> +
>  #include "log/logd/lgs_file.h"
>  #include "log/logd/lgs_filehdl.h"
> +#include "log/logd/lgs_imm.h"
> 
>  /***
>   * The following functions are used to handle a list of runtime stream
objects
> diff --git a/src/log/logd/lgs_stream.cc b/src/log/logd/lgs_stream.cc
> index 0c83e305b..4f50f5a98 100644
> --- a/src/log/logd/lgs_stream.cc
> +++ b/src/log/logd/lgs_stream.cc
> @@ -30,14 +30,16 @@
>  #include "log/logd/lgs_stream.h"
>  #include <algorithm>
> 
> +#include "base/osaf_time.h"
> +#include "osaf/immutil/immutil.h"
> +
>  #include "log/logd/lgs.h"
>  #include "log/logd/lgs_config.h"
>  #include "log/logd/lgs_file.h"
>  #include "log/logd/lgs_filehdl.h"
>  #include "log/logd/lgs_mbcsv_v1.h"
>  #include "log/logd/lgs_mbcsv_v2.h"
> -#include "base/osaf_time.h"
> -#include "osaf/immutil/immutil.h"
> +#include "log/logd/lgs_oi_admin.h"
> 
>  #define DEFAULT_NUM_APP_LOG_STREAMS 64
> 
> @@ -444,15 +446,18 @@ void log_stream_delete(log_stream_t **s) {
> 
>    if (lgs_cb->ha_state == SA_AMF_HA_ACTIVE) {
>      if (stream->isRtStream == SA_TRUE) {
> -      SaAisErrorT rv;
> +      SaAisErrorT ais_rc;
>        TRACE("Stream is closed, I am HA active so remove IMM object");
>        SaNameT objectName;
>        osaf_extended_name_lend(stream->name.c_str(), &objectName);
> -      rv = saImmOiRtObjectDelete(lgs_cb->immOiHandle, &objectName);
> -      if (rv != SA_AIS_OK) {
> +      ais_rc = saImmOiRtObjectDelete(lgsGetOiHandle(), &objectName);
> +      if (ais_rc != SA_AIS_OK) {
> +        if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +          lgsOiCreateBackground();
> +        }
>          /* no retry; avoid blocking LOG service #1886 */
> -        LOG_WA("saImmOiRtObjectDelete returned %u for %s", rv,
> -               stream->name.c_str());
> +        LOG_WA("%s: saImmOiRtObjectDelete returned %s for %s",
> +               __FUNCTION__, saf_error(ais_rc), stream->name.c_str());
>        }
>      }
>    }
> @@ -514,6 +519,53 @@ int lgs_populate_log_stream(
>    return rc;
>  }
> 
> +// Wrap creation of a runtime object. Handle try again ERR_EXIST
> +// If the object already exist it is deleted and then the new object is
created
> +// Note: This is a help function to be used in createRtObject() only
> +// Note: immutil is used
> +// The first 4 parameters are the same as for
> immutil_saImmOiRtObjectCreate_2
> +// The last parameter is the same as object name used with
> +// immutil_saImmOiRtObjectDelete()
> +static SaAisErrorT createRtObject(const SaImmOiHandleT oi_handle,
> +                                  const SaImmClassNameT class_name,
> +                                  const SaNameT* parent_name,
> +                                  const SaImmAttrValuesT_2**
attribute_values,
> +                                  const SaNameT *object_name) {
> +SaAisErrorT ais_rc = SA_AIS_OK;
> +
> +  for (int i = 0; i <= 2; i++) {
> +    ais_rc = immutil_saImmOiRtObjectCreate_2(oi_handle,
> +                                             class_name,
> +                                             parent_name,
> +                                             attribute_values);
> +    if (ais_rc == SA_AIS_ERR_EXIST) {
> +      SaAisErrorT del_rc = immutil_saImmOiRtObjectDelete(oi_handle,
> object_name);
> +      if (del_rc == SA_AIS_ERR_BAD_HANDLE) {
> +        LOG_NO("%s: saImmOiRtObjectDelete() Fail, %s", __FUNCTION__,
> +               saf_error(del_rc));
> +        lgsOiCreateBackground();
> +      } else if (del_rc != SA_AIS_OK) {
> +        LOG_NO("%s: saImmOiRtObjectDelete() Fail, %s", __FUNCTION__,
> +               saf_error(del_rc));
> +        break;
> +      } else {
> +        // The existing object is deleted. Try again to create a new one
> +        continue;
> +      }
> +    } else if (ais_rc == SA_AIS_ERR_BAD_HANDLE) {
> +      LOG_NO("%s: saImmOiRtObjectCreate_2() Fail, %s",
> +             __FUNCTION__, saf_error(ais_rc));
> +      lgsOiCreateBackground();
> +      break;
> +    } else {
> +      break;
> +    }
> +  }
> +
> +  // saImmOiRtObjectCreate_2() return code
> +  return ais_rc;
> +}
> +
>  /**
>   * Create a new rt app stream object.
>   *
> @@ -536,8 +588,9 @@ SaAisErrorT
> lgs_create_appstream_rt_object(log_stream_t *const stream) {
>        parent_name++; /* FIX, vulnerable for malformed DNs */
>        parentName = &parent;
>        osaf_extended_name_lend(parent_name, &parent);
> -    } else
> +    } else {
>        rdnstr = const_cast<char *>(stream->name.c_str());
> +    }
> 
>      void *arr1[] = {&rdnstr};
>      const SaImmAttrValuesT_2 attr_safLgStr = {
> @@ -622,14 +675,20 @@ SaAisErrorT
> lgs_create_appstream_rt_object(log_stream_t *const stream) {
>          &attr_saLogStreamCreationTimestamp,
>          NULL};
> 
> -    rc = immutil_saImmOiRtObjectCreate_2(
> -        lgs_cb->immOiHandle,
> const_cast<SaImmClassNameT>("SaLogStream"),
> -        parentName, attrValues);
> +    SaNameT object_name;
> +    osaf_extended_name_lend(stream->name.c_str(), &object_name);
> +    SaImmOiHandleT oi_handle = lgsGetOiHandle();
> +    rc = createRtObject(oi_handle,
> +                        const_cast<SaImmClassNameT>("SaLogStream"),
> +                        parentName,
> +                        attrValues,
> +                        &object_name);
>      free(dndup);
> 
>      if (rc != SA_AIS_OK) {
> -      LOG_WA("saImmOiRtObjectCreate_2 returned %u for %s, parent %s", rc,
> -             stream->name.c_str(), parent_name);
> +      LOG_WA("%s: createRtObject() returned %s for %s, "
> +             "parent %s", __FUNCTION__, saf_error(rc),
stream->name.c_str(),
> +             parent_name);
>      }
>    }
> 
> --
> 2.16.2



------------------------------------------------------------------------------
Check out the vibrant tech community on one of the world's most
engaging tech sites, Slashdot.org! http://sdm.link/slashdot
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel

Reply via email to