Hi Rafel,

Tested the patch, but following are the comments:

1. when the controllers are included

immcfg -a numberOfSingleSteps=2  -a nodesForSingleStep=SC-1 -a 
nodesForSingleStep=SC-2  openSafSmfExecControl=MergeCampToSS2

I have seen the segmentation fault.

Sep  2 17:10:03 SLES1 osafsmfd[2551]: ER SmfProcedureThread::init, 
getImmSteps FAILED, rc=SA_AIS_ERR_NOT_EXIST (12)
Sep  2 17:10:03 SLES1 osafsmfd[2551]: ER SmfProcedureThread::main, 
SmfProcedureThread: init failed
Sep  2 17:10:03 SLES1 osafsmfd[2551]: ER Start of procedure thread 
failed for 
safSmfProc=SmfBalancedProc0,safSmfCampaign=Campaign1,safApp=safSmfService
Sep  2 17:10:03 SLES1 osafimmnd[2459]: NO Implementer disconnected 11 
<336, 2010f> (safSmfProc1)
Sep  2 17:10:03 SLES1 osafimmnd[2459]: NO Implementer disconnected 12 
<411, 2010f> (safSmfProc2)
Sep  2 17:10:03 SLES1 osafimmnd[2459]: NO Implementer disconnected 15 
<480, 2010f> (safSmfProc3)
Sep  2 17:10:03 SLES1 osafamfnd[2520]: NO 
'safComp=SMF,safSu=SC-1,safSg=2N,safApp=OpenSAF' faulted due to 
'avaDown' : Recovery is 'nodeFailfast'
Sep  2 17:10:03 SLES1 osafamfnd[2520]: ER 
safComp=SMF,safSu=SC-1,safSg=2N,safApp=OpenSAF Faulted due to:avaDown 
Recovery is:nodeFailfast
Sep  2 17:10:03 SLES1 osafamfnd[2520]: Rebooting OpenSAF NodeId = 131343 
EE Name = , Reason: Component faulted: recovery is node failfast, 
OwnNodeId = 131343, SupervisionTime = 60
Sep  2 17:10:03 SLES1 kernel: [  163.790156] osafsmfd[2675]: segfault at 
51 ip 00007f1c89310b08 sp 00007f1c8bd968c0 error 4 in 
libgcc_s.so.1[7f1c89301000+16000]
Sep  2 17:05:46 SLES1 osafimmnd[2483]: NO Implementer locally 
disconnected. Marking it as doomed 16 <546, 2010f> (safSmfProc4)
Sep  2 17:05:46 SLES1 osafimmnd[2483]: NO Implementer locally 
disconnected. Marking it as doomed 9 <314, 2010f> (safSmfService)
Sep  2 17:05:46 SLES1 osafimmnd[2483]: NO Implementer locally 
disconnected. Marking it as doomed 10 <326, 2010f> (safSmfCampaign)
Sep  2 17:05:46 SLES1 osafimmnd[2483]: NO Implementer disconnected 16 
<546, 2010f> (safSmfProc4)
Sep  2 17:05:46 SLES1 osafimmnd[2483]: NO Implementer disconnected 9 
<314, 2010f> (safSmfService)
Sep  2 17:05:46 SLES1 opensaf_reboot: Rebooting local node; timeout=60


bt(full bt i did not get):

signal: 11 pid: 96 uid: 0
/usr/lib/../lib64/libopensaf_core.so.0(+0x1dfed)[0x7f69a5b42fed]
/lib64/libpthread.so.0(+0xf7c0)[0x7f69a481f7c0]
/usr/lib64/opensaf/osafsmfd[0x429539]
/usr/lib64/opensaf/osafsmfd[0x42b5dd]
/usr/lib64/opensaf/osafsmfd[0x410801]
/usr/lib64/opensaf/osafsmfd[0x410a38]
/usr/lib64/opensaf/osafsmfd[0x40aae3]
/usr/lib64/opensaf/osafsmfd(_ZN17SmfCampaignThread4mainEPv+0x32)[0x40abe2]
/lib64/libpthread.so.0(+0x77b6)[0x7f69a48177b6]
/lib64/libc.so.6(clone+0x6d)[0x7f69a37bb9cd]

smfd traces:
Sep  2 17:10:03.789791 osafsmfd [2551:SmfProcedureThread.cc:0060] TR 
Procedure thread exits
Sep  2 17:10:03.789799 osafsmfd [2551:SmfProcedureThread.cc:0088] >> 
~SmfProcedureThread
Sep  2 17:10:03.789836 osafsmfd [2551:SmfProcedureThread.cc:0096] << 
~SmfProcedureThread
Sep  2 17:10:03.789872 osafsmfd [2551:SmfProcedureThread.cc:0178] << stop
Sep  2 17:10:03.789888 osafsmfd [2551:SmfUpgradeProcedure.cc:0159] << 
~SmfUpgradeProcedure
Sep  2 17:10:03.789897 osafsmfd [2551:SmfUpgradeProcedure.cc:0121] >> 
~SmfUpgradeProcedure
Sep  2 17:10:03.789905 osafsmfd [2551:SmfUpgradeProcedure.cc:0159] << 
~SmfUpgradeProcedure
Sep  2 17:10:03.789913 osafsmfd [2551:SmfUpgradeProcedure.cc:0121] >> 
~SmfUpgradeProcedure
Sep  2 17:10:03.789921 osafsmfd [2551:SmfUpgradeProcedure.cc:0159] << 
~SmfUpgradeProcedure
Sep  2 17:10:03.789929 osafsmfd [2551:SmfUpgradeCampaign.cc:0103] << 
~SmfUpgradeCampaign
Sep  2 17:10:03.789959 osafsmfd [2551:SmfUpgradeCampaign.cc:0796] >> execute


2.
Is this enhancement , considers only PLs for nodesForSingleStep? If yes, 
then SCs should not be allowed.
There must be check, to allow only PLs.

Regards,
Neel.



On 2016/09/01 02:27 PM, Rafael Odzakow wrote:
>   osaf/services/saf/smfsv/config/smfsv_classes.xml    |   13 +
>   osaf/services/saf/smfsv/smfd/Makefile.am            |    6 +-
>   osaf/services/saf/smfsv/smfd/SmfCampState.cc        |  126 +-----
>   osaf/services/saf/smfsv/smfd/SmfCampaign.cc         |   77 ++-
>   osaf/services/saf/smfsv/smfd/SmfCampaign.hh         |    1 +
>   osaf/services/saf/smfsv/smfd/SmfExecControl.cc      |  410 
> ++++++++++++++++++++
>   osaf/services/saf/smfsv/smfd/SmfExecControl.h       |   59 ++
>   osaf/services/saf/smfsv/smfd/SmfProcState.cc        |   28 +-
>   osaf/services/saf/smfsv/smfd/SmfUpgradeCampaign.cc  |   32 +-
>   osaf/services/saf/smfsv/smfd/SmfUpgradeCampaign.hh  |   24 +-
>   osaf/services/saf/smfsv/smfd/SmfUpgradeProcedure.cc |   27 +-
>   osaf/services/saf/smfsv/smfd/SmfUpgradeProcedure.hh |   18 +-
>   osaf/services/saf/smfsv/smfd/SmfUpgradeStep.hh      |   13 +-
>   osaf/services/saf/smfsv/smfd/smfd.h                 |    2 +-
>   14 files changed, 653 insertions(+), 183 deletions(-)
>
>
> diff --git a/osaf/services/saf/smfsv/config/smfsv_classes.xml 
> b/osaf/services/saf/smfsv/config/smfsv_classes.xml
> --- a/osaf/services/saf/smfsv/config/smfsv_classes.xml
> +++ b/osaf/services/saf/smfsv/config/smfsv_classes.xml
> @@ -531,5 +531,18 @@
>                       <flag>SA_WRITABLE</flag>
>                       <default-value>0</default-value>
>               </attr>
> +             <attr>
> +                     <name>numberOfSingleSteps</name>
> +                     <type>SA_UINT32_T</type>
> +                     <category>SA_CONFIG</category>
> +                     <flag>SA_WRITABLE</flag>
> +             </attr>
> +             <attr>
> +                     <name>nodesForSingleStep</name>
> +                     <type>SA_STRING_T</type>
> +                     <category>SA_CONFIG</category>
> +      <flag>SA_WRITABLE</flag>
> +                     <flag>SA_MULTI_VALUE</flag>
> +             </attr>
>       </class>
>   </imm:IMM-contents>
> diff --git a/osaf/services/saf/smfsv/smfd/Makefile.am 
> b/osaf/services/saf/smfsv/smfd/Makefile.am
> --- a/osaf/services/saf/smfsv/smfd/Makefile.am
> +++ b/osaf/services/saf/smfsv/smfd/Makefile.am
> @@ -49,7 +49,8 @@ noinst_HEADERS = \
>       SmfCampaignWrapup.hh \
>       SmfCampaignInit.hh \
>       SmfCallback.hh \
> -     SmfCbkUtil.hh
> +     SmfCbkUtil.hh \
> +     SmfExecControl.h
>   
>   osafsmfd_CXXFLAGS = $(AM_CXXFLAGS) @XML2_CFLAGS@
>   
> @@ -87,7 +88,8 @@ osafsmfd_SOURCES = \
>       SmfCampaignInit.cc \
>       SmfCampaignWrapup.cc \
>       SmfCallback.cc \
> -     SmfCbkUtil.cc
> +     SmfCbkUtil.cc \
> +     SmfExecControl.cc
>   
>   osafsmfd_LDFLAGS = \
>       $(AM_LDFLAGS) \
> diff --git a/osaf/services/saf/smfsv/smfd/SmfCampState.cc 
> b/osaf/services/saf/smfsv/smfd/SmfCampState.cc
> --- a/osaf/services/saf/smfsv/smfd/SmfCampState.cc
> +++ b/osaf/services/saf/smfsv/smfd/SmfCampState.cc
> @@ -32,8 +32,10 @@
>   #include "SmfCampaignThread.hh"
>   #include "SmfCampaign.hh"
>   #include "SmfProcedureThread.hh"
> +#include "smfsv_defs.h"
>   #include <immutil.h>
>   #include <sstream>
> +#include "SmfUpgradeAction.hh"
>   /* ========================================================================
>    *   DEFINITIONS
>    * ========================================================================
> @@ -764,12 +766,7 @@ SmfCampStateExecuting::execute(SmfUpgrad
>       //must be restarted. The execution shall continue at step execution 
> phase. The procedure initialization
>       //and step calculation was performed before the move of control.
>   
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -                procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>           std::vector < SmfUpgradeProcedure * >::iterator iter;
>       bool execProcFound = false;
> @@ -817,13 +814,7 @@ SmfCampStateExecuting::executeProc(SmfUp
>       //The procedure vector is sorted in execution level order (low -> high)
>       //Lowest number shall be executed first.
>   
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -                procedures = i_camp->getProcedures();
> -        }
> -
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>       std::vector < SmfUpgradeProcedure * >::iterator iter;
>       int execLevel = -1;
>   
> @@ -912,12 +903,7 @@ SmfCampStateExecuting::suspend(SmfUpgrad
>       TRACE("SmfCampStateExecuting::suspend implementation");
>   
>       /* Send suspend message to all procedures */
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -                procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>       std::vector < SmfUpgradeProcedure * >::iterator iter;
>   
> @@ -966,12 +952,7 @@ SmfCampStateExecuting::procResult(SmfUpg
>                   LOG_NO("CAMP: Procedure %s returned STEPUNDONE", 
> i_procedure->getProcName().c_str());
>   
>                   /* Send suspend message to all procedures */
> -                std::vector < SmfUpgradeProcedure * > procedures;
> -                if (i_camp->getProcExecutionMode() == 
> SMF_MERGE_TO_SINGLE_STEP) {
> -                        procedures.push_back(i_camp->getMergedProc());
> -                } else {
> -                        procedures = i_camp->getProcedures();
> -                }
> +                std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>               std::vector < SmfUpgradeProcedure * >::iterator iter;
>               for (iter = procedures.begin(); iter != procedures.end(); 
> ++iter) {
> @@ -1114,21 +1095,8 @@ SmfCampStateExecCompleted::commit(SmfUpg
>           i_camp->resetMaintenanceState(); // No action if it fails
>   
>           //Remove the procedure runtime objects
> -       if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                i_camp->getMergedProc()->commit();
> -
> -        } else {
> -               std::vector < SmfUpgradeProcedure * > procedures;
> -               if (i_camp->getProcExecutionMode() == 
> SMF_MERGE_TO_SINGLE_STEP) {
> -                       procedures.push_back(i_camp->getMergedProc());
> -               } else {
> -                       procedures = i_camp->getProcedures();
> -               }
> -
> -                std::vector < SmfUpgradeProcedure * >::iterator iter;
> -                for (iter = procedures.begin(); iter != procedures.end(); 
> ++iter) {
> -                        (*iter)->commit();
> -                }
> +        for (auto proc: i_camp->getProcedures()) {
> +            proc->commit();
>           }
>   
>           i_camp->removeRunTimeObjects(); // No action if it fails
> @@ -1213,12 +1181,7 @@ SmfCampStateSuspendingExec::execute(SmfU
>       //in while in SuspendingExec state
>       i_camp->setProcExecutionMode(smfd_cb->procExecutionMode);
>   
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -                procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>       std::vector < SmfUpgradeProcedure * >::iterator iter;
>       bool initialFound = false;
> @@ -1358,12 +1321,7 @@ SmfCampStateExecSuspended::execute(SmfUp
>       i_camp->setProcExecutionMode(smfd_cb->procExecutionMode);
>   
>       /* Send execute to all suspended procedures */
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -                procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>       std::vector < SmfUpgradeProcedure * >::iterator iter;
>   
> @@ -1423,12 +1381,7 @@ SmfCampStateExecSuspended::rollback(SmfU
>       }
>   
>       /* Send rollback to all suspended procedures */
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -                procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>       std::vector < SmfUpgradeProcedure * >::iterator iter;
>   
> @@ -1728,12 +1681,7 @@ SmfCampStateSuspendedByErrorDetected::ex
>       i_camp->setProcExecutionMode(smfd_cb->procExecutionMode);
>   
>       /* Send execute to all suspended/undone procedures */
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -                procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>       std::vector < SmfUpgradeProcedure * >::iterator iter;
>   
> @@ -1789,12 +1737,7 @@ SmfCampStateSuspendedByErrorDetected::ro
>       }
>   
>       /* Send rollback to all suspended/undone procedures */
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -                procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>       std::vector < SmfUpgradeProcedure * >::iterator iter;
>   
> @@ -1873,12 +1816,7 @@ SmfCampRollingBack::rollback(SmfUpgradeC
>       TRACE_ENTER();
>       TRACE("SmfCampRollingBack::rollback implementation");
>   
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -                procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>       std::vector < SmfUpgradeProcedure * >::reverse_iterator iter;
>   
> @@ -1960,12 +1898,7 @@ SmfCampRollingBack::rollbackProc(SmfUpgr
>       //The procedure vector is sorted in execution level order (low -> high)
>       //Highest number shall be rolled back first so start from end of list.
>   
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -                procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>       std::vector < SmfUpgradeProcedure * >::reverse_iterator iter;
>   
> @@ -2017,7 +1950,8 @@ SmfCampRollingBack::rollbackSingleMergeP
>   {
>       TRACE_ENTER();
>       LOG_NO("CAMP:: Rollback merged single step procedure only");
> -        SmfUpgradeProcedure * mergedProc = i_camp->getMergedProc();
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
> +        SmfUpgradeProcedure * mergedProc = procedures.at(0);
>           if (mergedProc->getState() == SA_SMF_PROC_COMPLETED) {
>                   SmfProcedureThread *procThread = 
> mergedProc->getProcThread();
>                   PROCEDURE_EVT *evt = new PROCEDURE_EVT();
> @@ -2046,12 +1980,7 @@ SmfCampRollingBack::suspend(SmfUpgradeCa
>       TRACE("SmfCampRollingBack::suspend implementation");
>   
>       /* Send suspend message to all procedures */
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -          procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -          procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>       std::vector < SmfUpgradeProcedure * >::iterator iter;
>   
> @@ -2172,12 +2101,7 @@ SmfCampRollbackSuspended::rollback(SmfUp
>       TRACE("SmfCampRollbackSuspended::rollback implementation");
>   
>       /* Send rollback to all suspended procedures */
> -        std::vector < SmfUpgradeProcedure * > procedures;
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -          procedures.push_back(i_camp->getMergedProc());
> -        } else {
> -          procedures = i_camp->getProcedures();
> -        }
> +        std::vector<SmfUpgradeProcedure*> procedures = 
> i_camp->getProcedures();
>   
>       std::vector < SmfUpgradeProcedure * >::iterator iter;
>   
> @@ -2345,16 +2269,8 @@ SmfCampRollbackCompleted::commit(SmfUpgr
>           i_camp->resetMaintenanceState(); // No action if it fails
>   
>           //Remove the procedure runtime objects
> -        if (i_camp->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> -                i_camp->getMergedProc()->commit();
> -
> -        } else {
> -                const std::vector < SmfUpgradeProcedure * >& procedures = 
> i_camp->getProcedures();
> -                std::vector < SmfUpgradeProcedure * >::const_iterator iter;
> -
> -                for (iter = procedures.begin(); iter != procedures.end(); 
> ++iter) {
> -                        (*iter)->commit();
> -                }
> +        for (auto proc: i_camp->getProcedures()) {
> +            proc->commit();
>           }
>           
>           i_camp->removeRunTimeObjects(); // No action if it fails
> diff --git a/osaf/services/saf/smfsv/smfd/SmfCampaign.cc 
> b/osaf/services/saf/smfsv/smfd/SmfCampaign.cc
> --- a/osaf/services/saf/smfsv/smfd/SmfCampaign.cc
> +++ b/osaf/services/saf/smfsv/smfd/SmfCampaign.cc
> @@ -18,6 +18,7 @@
>   #include <sys/stat.h>
>   #include <new>
>   #include <vector>
> +#include <algorithm>
>   #include <string>
>   #include <sys/time.h>
>   
> @@ -30,7 +31,9 @@
>   #include "SmfUpgradeProcedure.hh"
>   #include "SmfUpgradeMethod.hh"
>   #include "SmfProcedureThread.hh"
> +#include "SmfUpgradeAction.hh"
>   #include "SmfUtils.hh"
> +#include "SmfExecControl.h"
>   
>   #include "saAis.h"
>   #include <saSmf.h>
> @@ -661,6 +664,10 @@ SmfCampaign::initExecution(void)
>   SaAisErrorT
>   SmfCampaign::startProcedureThreads()
>   {
> +        // If any procedure start goes badly two things happen: delete of the
> +        // upgrade campaign pointer to terminate and remove previously 
> started
> +        // procedures. Return SA_AIS_OK to not change campaign state and 
> allow
> +        // reexecution
>           TRACE_ENTER();
>           SmfUpgradeCampaign *p_uc = getUpgradeCampaign();
>           if (p_uc->getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) {
> @@ -676,25 +683,25 @@ SmfCampaign::startProcedureThreads()
>                   std::string singleProcDN = singleProc->getProcName() + "," +
>                           SmfCampaignThread::instance()->campaign()->getDn();
>                   singleProc->setDn(singleProcDN);
> -                p_uc->setMergedProc(singleProc);
> +                p_uc->addMergedProcedure(singleProc);
>   
> -                /* Start procedure thread */
> -                SmfProcedureThread *procThread = new 
> SmfProcedureThread(singleProc);
> -                /* The procThread will set itself when started correctly */
> -                singleProc->setProcThread(NULL);
> -
> -                LOG_NO("SmfCampaign::startProcedureThreads, Starting 
> procedure thread %s",
> -                       singleProc->getProcName().c_str());
> -                procThread->start();
> -                /* Check if procedure thread started correctly */
> -                if (singleProc->getProcThread() == NULL) {
> -                        std::string error = "Start of procedure thread 
> failed for " + singleProcDN;
> -                        LOG_ER("%s", error.c_str());
> -                        
> SmfCampaignThread::instance()->campaign()->setError(error);
> -                        delete p_uc; // To terminate and remove any 
> previously started procedure threads
> -                        /* Don't change campaign state to allow reexecution 
> */
> +                if (startProcedure(singleProc) == false) {
> +                        delete p_uc;
>                           return SA_AIS_OK;
>                   }
> +        } else if (p_uc->getProcExecutionMode() == SMF_BALANCED_MODE) {
> +                if (!execctrl::createBalancedProcs()) {
> +                        return SA_AIS_ERR_INIT;
> +                }
> +                auto procedures = p_uc->getProcedures();
> +                TRACE("SmfCampaign::startProcedureThreads, number of 
> procedures=[%zu]", procedures.size());
> +                for (auto proc: procedures) {
> +                        proc->setDn(proc->getProcName() + "," + 
> SmfCampaignThread::instance()->campaign()->getDn());
> +                        if (startProcedure(proc) == false) {
> +                                delete p_uc;
> +                                return SA_AIS_OK;
> +                        }
> +                }
>           } else {
>                   const std::vector < SmfUpgradeProcedure * >& procedures = 
> p_uc->getProcedures();
>                   std::vector < SmfUpgradeProcedure * >::const_iterator iter;
> @@ -718,24 +725,10 @@ SmfCampaign::startProcedureThreads()
>                           }
>                           (*iter)->setDn(dn);
>   
> -                        /* Start procedure thread */
> -                        SmfProcedureThread *procThread = new 
> SmfProcedureThread(*iter);
> -                        /* The procThread will set itself when started 
> correctly */
> -                        (*iter)->setProcThread(NULL);
> -
> -                        TRACE("SmfCampaign::startProcedureThreads, Starting 
> procedure thread %s", (*iter)->getProcName().c_str());
> -                        procThread->start();
> -
> -                        /* Check if procedure thread started correctly */
> -                        if ((*iter)->getProcThread() == NULL) {
> -                                std::string error = "Start of procedure 
> thread failed for " + dn;
> -                                LOG_ER("%s", error.c_str());
> -                                
> SmfCampaignThread::instance()->campaign()->setError(error);
> -                                delete p_uc; // To terminate and remove any 
> previously started procedure threads
> -                                /* Don't change campaign state to allow 
> reexecution */
> +                        if (startProcedure(*iter) == false) {
> +                                delete p_uc;
>                                   return SA_AIS_OK;
>                           }
> -
>                           iter++;
>                   }
>           }
> @@ -743,6 +736,26 @@ SmfCampaign::startProcedureThreads()
>           return SA_AIS_OK; //Will never return here, just for compiler
>   }
>   
> +//--------------------------------------------------------------------------
> +// Starts one upgrade procedure thread, return false on failure
> +//--------------------------------------------------------------------------
> +bool
> +SmfCampaign::startProcedure(SmfUpgradeProcedure* procedure) {
> +        SmfProcedureThread *procedure_thread = new 
> SmfProcedureThread(procedure);
> +        // The procThread will set itself when started correctly
> +        procedure->setProcThread(nullptr);
> +        TRACE("SmfCampaign::startProcedure dn: %s", 
> procedure->getDn().c_str());
> +        procedure_thread->start();
> +
> +        if (procedure->getProcThread() == nullptr) {
> +                std::string error = "Start of procedure thread failed for " 
> + procedure->getDn();
> +                LOG_ER("%s", error.c_str());
> +                SmfCampaignThread::instance()->campaign()->setError(error);
> +                return false;
> +        }
> +        return true;
> +}
> +
>   /**
>    * procResult
>    * Takes care of procedure result
> diff --git a/osaf/services/saf/smfsv/smfd/SmfCampaign.hh 
> b/osaf/services/saf/smfsv/smfd/SmfCampaign.hh
> --- a/osaf/services/saf/smfsv/smfd/SmfCampaign.hh
> +++ b/osaf/services/saf/smfsv/smfd/SmfCampaign.hh
> @@ -107,6 +107,7 @@ class SmfCampaign {
>           void stopElapsedTime();
>   
>    private:
> +        bool startProcedure(SmfUpgradeProcedure* procedure);
>   
>       std::string m_dn;
>       std::string m_cmpg;
> diff --git a/osaf/services/saf/smfsv/smfd/SmfExecControl.cc 
> b/osaf/services/saf/smfsv/smfd/SmfExecControl.cc
> new file mode 100644
> --- /dev/null
> +++ b/osaf/services/saf/smfsv/smfd/SmfExecControl.cc
> @@ -0,0 +1,410 @@
> +/*
> + *
> + * (C) Copyright 2009 The OpenSAF Foundation
> + *
> + * This program is distributed in the hope that it will be useful, but
> + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
> + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed
> + * under the GNU Lesser General Public License Version 2.1, February 1999.
> + * The complete license can be accessed from the following location:
> + * http://opensource.org/licenses/lgpl-license.php
> + * See the Copying file included with the OpenSAF distribution for full
> + * licensing terms.
> + *
> + * Author(s): Ericsson AB
> + *
> + */
> +
> +#include "SmfExecControl.h"
> +#include <list>
> +#include <limits>
> +#include <algorithm>
> +#include "logtrace.h"
> +#include "immutil.h"
> +#include "SmfUpgradeProcedure.hh"
> +#include "SmfUpgradeMethod.hh"
> +#include "SmfUtils.hh"
> +#include "SmfUpgradeAction.hh"
> +#include "SmfUpgradeCampaign.hh"
> +
> +namespace execctrl {
> +
> +static std::vector<SmfUpgradeStep*> 
> getStepsMatchingBalancedGroup(SmfUpgradeProcedure* procedure);
> +static void addInitAndWrapupActionsToProcedure(SmfUpgradeProcedure* 
> procedure,
> +    const std::vector<SmfUpgradeAction*>& initactions,
> +    const std::vector<SmfUpgradeAction*>& wrapupactions);
> +static SmfUpgradeStep* mergeStep(SmfUpgradeProcedure* procedure,
> +    const std::vector<SmfUpgradeStep*>& steps);
> +static bool readExecControlObject(unsigned int* numberOfSingleSteps,
> +                           std::vector<std::string>* nodesForSingleStep);
> +static bool removeDuplicateActivationUnits(SmfUpgradeProcedure * i_newproc,
> +                                    SmfUpgradeStep *newStep);
> +static bool isNodeInGroup(const std::string& node,
> +                          const std::vector<std::string>& group) {
> +  return std::find(group.begin(), group.end(), node) != group.end();
> +}
> +
> +bool createBalancedProcs() {
> +  // Creates new procedures based on the ExecControl configuration
> +  TRACE_ENTER();
> +  unsigned int numberofss;
> +  std::vector<std::string> nodesforss;
> +  std::vector<SmfUpgradeProcedure*> balancedprocs;
> +  // Assume that nodesForSingleStep only contains nodes used by the campaign.
> +  if (!readExecControlObject(&numberofss, &nodesforss)) {
> +    return false;
> +  }
> +  // chunk is the size of the balanced group
> +  unsigned int chunk = (nodesforss.size() + numberofss - 1) / numberofss;
> +  TRACE("balanced group size will be %i", chunk);
> +  std::vector<std::string>::iterator itr;
> +  std::vector<std::string>::iterator iend;
> +  // Set the procedure exec level to be as high as possible
> +  int procExecLvl = std::numeric_limits<int>::max() - numberofss - 1;
> +  int procnum = 0;
> +  for (itr = nodesforss.begin(); iend < nodesforss.end(); itr += chunk) {
> +    iend = itr + chunk;
> +    SmfUpgradeProcedure *ssproc = new(std::nothrow) SmfUpgradeProcedure;
> +    if (iend >= nodesforss.end()) {
> +      iend = nodesforss.end();
> +    }
> +    ssproc->setUpgradeMethod(new(std::nothrow) SmfSinglestepUpgrade);
> +    ssproc->setProcName("safSmfProc=SmfBalancedProc" + 
> std::to_string(procnum));
> +    ssproc->setExecLevel(std::to_string(procExecLvl));
> +    ssproc->setIsMergedProcedure(true);  // For cleanup purposes
> +    // Each new procedure holds the balanced group it steps over
> +    ssproc->setBalancedGroup(std::vector<std::string>(itr, iend));
> +    balancedprocs.push_back(ssproc);
> +    procExecLvl++;
> +    procnum++;
> +  }
> +  for (auto proc : balancedprocs) {
> +      auto camp = SmfCampaignThread::instance()->campaign();
> +      camp->getUpgradeCampaign()->addUpgradeProcedure(proc);
> +  }
> +  TRACE_LEAVE();
> +  return true;
> +}
> +
> +bool createStepForBalancedProc(SmfUpgradeProcedure* procedure) {
> +  TRACE_ENTER();
> +  std::vector<SmfUpgradeStep*> steps;
> +  std::vector<SmfUpgradeAction*> initactions;
> +  std::vector<SmfUpgradeAction*> wrapupactions;
> +  for (auto step : getStepsMatchingBalancedGroup(procedure)) {
> +    // copy the steps together with actions and callbacks
> +    auto oproc = step->getProcedure();
> +    steps.insert(steps.end(),
> +                 oproc->getProcSteps().begin(),
> +                 oproc->getProcSteps().end());
> +    initactions.insert(initactions.end(),
> +                       oproc->getInitActions().begin(),
> +                       oproc->getInitActions().end());
> +    wrapupactions.insert(wrapupactions.end(),
> +                         oproc->getWrapupActions().begin(),
> +                         oproc->getWrapupActions().end());
> +    procedure->getCallbackList(oproc->getUpgradeMethod());
> +  }
> +  if (!steps.empty()) {
> +    SmfUpgradeStep* newstep = mergeStep(procedure, steps);
> +    addInitAndWrapupActionsToProcedure(procedure, initactions, 
> wrapupactions);
> +    removeDuplicateActivationUnits(procedure, newstep);
> +    procedure->addProcStep(newstep);
> +  }
> +  TRACE_LEAVE();
> +  return true;
> +}
> +
> +bool allStepsMerged(SmfUpgradeProcedure* procedure) {
> +  TRACE_ENTER();
> +  if (!procedure->getBalancedGroup().empty()) {
> +    TRACE("not an original proc");
> +    return false;
> +  }
> +  auto camp = SmfCampaignThread::instance()->campaign();
> +  auto allprocs = camp->getUpgradeCampaign()->getProcedures();
> +
> +  // Build the "nodesForSingleStep" from all balanced groups
> +  std::vector<std::string> groups;
> +  for (auto proc : allprocs) {
> +    if (!proc->getBalancedGroup().empty()) {
> +      groups.insert(groups.begin(),
> +                    proc->getBalancedGroup().begin(),
> +                    proc->getBalancedGroup().end());
> +    }
> +  }
> +
> +  bool allmerged = true;
> +  for (auto step : procedure->getProcSteps()) {
> +    if (!isNodeInGroup(step->getSwNode(), groups)) {
> +      TRACE("node not in group stepNode:%s", step->getSwNode().c_str());
> +      allmerged = false;
> +    } else {
> +      TRACE("node in group stepNode:%s", step->getSwNode().c_str());
> +      step->setStepState(SA_SMF_STEP_COMPLETED);
> +    }
> +  }
> +  TRACE("allStepsMerged returns:%d", allmerged);
> +  TRACE_LEAVE();
> +  return allmerged;
> +}
> +
> +void trace(const std::string& message) {
> +  TRACE("-tracing procedures in UpgradeCampaign:%s", message.c_str());
> +  auto camp = SmfCampaignThread::instance()->campaign();
> +  for (auto proc : camp->getUpgradeCampaign()->getProcedures()) {
> +    TRACE("-procedure: name:%s dn:%s", proc->getName().c_str(), 
> proc->getDn().c_str());
> +    for (auto step : proc->getProcSteps()) {
> +      TRACE("    -step : %s, state %i", step->getDn().c_str(), 
> step->getState());
> +    }
> +    for (auto node : proc->getBalancedGroup()) {
> +      TRACE("    balanced node : %s", node.c_str());
> +    }
> +    for (auto iact : proc->getInitActions()) {
> +      auto cba = dynamic_cast<SmfCallbackAction*>(iact);
> +      TRACE("    iact cb label : %s", 
> cba->getCallback().getCallbackLabel().c_str());
> +    }
> +  }
> +}
> +
> +std::vector<SmfUpgradeStep*> getStepsMatchingBalancedGroup(
> +    SmfUpgradeProcedure* procedure) {
> +  TRACE_ENTER();
> +  // For all original procedures check if the steps are in the balanced group
> +  // of the procedure. Return the matching steps.
> +  std::vector<SmfUpgradeStep*> steps;
> +  auto camp = SmfCampaignThread::instance()->campaign();
> +  for (auto proc : camp->getUpgradeCampaign()->getProcedures()) {
> +    if (proc->getBalancedGroup().empty()) {
> +      for (auto ostep : proc->getProcSteps()) {
> +        if (isNodeInGroup(ostep->getSwNode(), 
> procedure->getBalancedGroup())) {
> +          TRACE("step in group: %s", ostep->getDn().c_str());
> +          steps.push_back(ostep);
> +        } else {
> +          TRACE("step not in group: %s", ostep->getDn().c_str());
> +        }
> +      }
> +    }
> +  }
> +  TRACE_LEAVE();
> +  return steps;
> +}
> +
> +void addInitAndWrapupActionsToProcedure(SmfUpgradeProcedure* procedure,
> +    const std::vector<SmfUpgradeAction*>& initactions,
> +    const std::vector<SmfUpgradeAction*>& wrapupactions) {
> +  // Add Init/WrapupActions to the procedure. The Actions themself contain a
> +  // pointer to the procedure.
> +  TRACE_ENTER();
> +  procedure->addInitActions(initactions);
> +  procedure->addWrapupActions(wrapupactions);
> +  for (auto iac : initactions) {
> +    const SmfCallbackAction* cba = dynamic_cast<const 
> SmfCallbackAction*>(iac);
> +    if (cba != nullptr) {
> +      const_cast<SmfCallbackAction*>(cba)->setCallbackProcedure(procedure);
> +    }
> +  }
> +  for (auto wac : wrapupactions) {
> +    const SmfCallbackAction* cba = dynamic_cast<const 
> SmfCallbackAction*>(wac);
> +    if (cba != nullptr) {
> +      const_cast<SmfCallbackAction*>(cba)->setCallbackProcedure(procedure);
> +    }
> +  }
> +  TRACE_LEAVE();
> +}
> +
> +SmfUpgradeStep* mergeStep(SmfUpgradeProcedure* procedure,
> +    const std::vector<SmfUpgradeStep*>& steps) {
> +  // Create a merged step based on the upgrade step passed in. The in/out
> +  // parameter procedure must be a balanced procedure.
> +  TRACE_ENTER();
> +  SmfUpgradeStep* newstep = new(std::nothrow)SmfUpgradeStep;
> +  osafassert(newstep != nullptr);
> +  newstep->setRdn("safSmfStep=0001");
> +  newstep->setDn(newstep->getRdn() + "," + procedure->getDn());
> +  newstep->setMaxRetry(0);
> +  newstep->setRestartOption(0);
> +  std::list <unitNameAndState> deact;
> +  for (auto step : steps) {
> +    if (isNodeInGroup(step->getSwNode(), procedure->getBalancedGroup())) {
> +      TRACE("adding modifications, deact and bundle ref from node:%s",
> +            step->getSwNode().c_str());
> +      newstep->addModifications(step->getModifications());
> +      deact.insert(deact.end(),
> +                   step->getDeactivationUnitList().begin(),
> +                   step->getDeactivationUnitList().end());
> +      procedure->mergeBundleRefRollingToSingleStep(newstep, step);
> +    }
> +  }
> +  TRACE_LEAVE();
> +  return newstep;
> +}
> +
> +bool readExecControlObject(unsigned int* numberOfSingleSteps,
> +                           std::vector<std::string>* nodesForSingleStep) {
> +  // Read the attributes from the exec control object from the IMM. The OI
> +  // will prevent any changes once a campaign is running.
> +  SmfImmUtils immutil;
> +  SaImmAttrValuesT_2 **attributes;
> +  if (immutil.getObject(SMF_CONFIG_OBJECT_DN, &attributes) == false) {
> +    LOG_ER("Could not get SMF config object from IMM %s", 
> SMF_CONFIG_OBJECT_DN);
> +    return false;
> +  }
> +  const char* execdn = immutil_getStringAttr(
> +      (const SaImmAttrValuesT_2 **)attributes, OPENSAF_SMF_EXEC_CONTROL, 0);
> +  if (execdn == NULL || strcmp(execdn, "") == 0) {
> +    LOG_ER("Could not get %s attrs from SmfConfig", 
> OPENSAF_SMF_EXEC_CONTROL);
> +    return false;
> +  }
> +  if (immutil.getObject(execdn, &attributes) == false) {
> +    LOG_ER("Failed to get object from attribute %s", 
> OPENSAF_SMF_EXEC_CONTROL);
> +    return false;
> +  }
> +  SaAisErrorT rc;
> +  SaUint32T numnodes;
> +  rc = immutil_getAttrValuesNumber(const_cast<char*>("nodesForSingleStep"),
> +                                   (const SaImmAttrValuesT_2 **)attributes,
> +                                   &numnodes);
> +  if (rc != SA_AIS_OK) {
> +    LOG_ER("nodesForSingleStep not configured");
> +    return false;
> +  }
> +  const char* node;
> +  for (unsigned int i = 0; i < numnodes; i++) {
> +    node = immutil_getStringAttr((const SaImmAttrValuesT_2**)attributes,
> +                                      "nodesForSingleStep", i);
> +    if (node == NULL) {
> +        break;
> +    }
> +    nodesForSingleStep->push_back("safAmfNode=" + std::string(node) +
> +                                  ",safAmfCluster=myAmfCluster");
> +  }
> +  const SaUint32T* numss = immutil_getUint32Attr(
> +      (const SaImmAttrValuesT_2 **)attributes, "numberOfSingleSteps", 0);
> +  if (numss == NULL) {
> +    LOG_ER("could not read numberOfSingleSteps");
> +    return false;
> +  }
> +
> +  *numberOfSingleSteps = *numss;
> +
> +  TRACE("numberOfSingleSteps %i", *numberOfSingleSteps);
> +  for (auto node : *nodesForSingleStep) {
> +    TRACE("nodesForSingleStep %s", node.c_str());
> +  }
> +  osafassert(numberOfSingleSteps != 0);
> +  return true;
> +}
> +
> +bool removeDuplicateActivationUnits(SmfUpgradeProcedure * i_newproc,
> +                                                SmfUpgradeStep *newStep) {
> +  // Remove any (de)activation unit duplicates and add them to the step.
> +  // Activation and deactivation units are the same because rolling and
> +  // formodify is symetric.
> +  TRACE_ENTER();
> +  std::list < unitNameAndState > tmpDU;
> +  std::multimap<std::string, objectInst> objInstances;
> +  if (i_newproc->getImmComponentInfo(objInstances) == false) {
> +    TRACE("Config info from IMM could not be read");
> +    return false;
> +  }
> +
> +  // Remove DU duplicates
> +  tmpDU.sort(compare_du_part);
> +  tmpDU.unique(unique_du_part);
> +
> +  // Reduce the DU list, check if smaller scope is within bigger scope.
> +  std::pair<std::multimap<std::string, objectInst>::iterator,
> +            std::multimap<std::string, objectInst>::iterator> nodeName_mm;
> +  std::multimap<std::string, objectInst>::iterator iter;
> +
> +  std::list < unitNameAndState > nodeLevelDU;
> +  std::list < unitNameAndState >::iterator unit_iter;
> +  // Find DU on node level and save them in a separate list
> +  for (unit_iter = tmpDU.begin(); unit_iter != tmpDU.end();) {
> +    if ((*unit_iter).name.find("safAmfNode=") == 0) {  // DU is a node node
> +      // A node will never be optimized away, save it
> +      nodeLevelDU.push_back(*unit_iter);
> +      // Remove the node and update iterator
> +      unit_iter = tmpDU.erase(unit_iter);
> +    } else {
> +      ++unit_iter;
> +    }
> +  }
> +
> +  // For all found nodes, look if some other DU (comp/SU) is within scope
> +  // tmpDU contain all DU except the node level ones which was removed above
> +  // and saved in nodeLevelDU list
> +  std::list < unitNameAndState >::iterator itr;
> +  for (itr = nodeLevelDU.begin(); itr != nodeLevelDU.end(); ++itr) {
> +    // For all comp/SU found in the scope of the node.  Find out if any
> +    // remaining DU is within it.
> +    // Get all components/SU within the node
> +    nodeName_mm = objInstances.equal_range((*itr).name);
> +    for (iter = nodeName_mm.first;  iter != nodeName_mm.second;  ++iter) {
> +      // For all comp/SU found in the scope of the node.
> +      // Find out if any remaining DU is within it
> +      for (unit_iter = tmpDU.begin(); unit_iter != tmpDU.end();) {
> +        if ((*unit_iter).name == (*iter).second.suDN) {  // Check SU
> +          TRACE("[%s] is in scope of [%s], remove it from DU list",
> +                 (*unit_iter).name.c_str(), (*itr).name.c_str());
> +          // Remove the node and update iterator
> +          unit_iter = tmpDU.erase(unit_iter);
> +        } else if ((*unit_iter).name == (*iter).second.compDN) {  // Check 
> comp
> +          TRACE("[%s] is in scope of [%s], remove it from DU list",
> +                 (*unit_iter).name.c_str(), (*itr).name.c_str());
> +          // Remove the node and update iterator
> +          unit_iter = tmpDU.erase(unit_iter);
> +        } else {
> +          ++unit_iter;
> +        }
> +      }
> +    }
> +  }
> +
> +  // tmpDU contain all DU which was not in the scope of an included node Find
> +  // DU on SU level and save them in a separate list. Remove SU from tmpDU 
> list
> +  std::list < unitNameAndState > suLevelDU;
> +  for (unit_iter = tmpDU.begin(); unit_iter != tmpDU.end();) {
> +    if ((*unit_iter).name.find("safSu=") == 0) {  // DU is a SU
> +      // A node will never be optimized away, save it
> +      suLevelDU.push_back(*unit_iter);
> +      unit_iter = tmpDU.erase(unit_iter);  // Remove the SU and update 
> iterator
> +    } else {
> +      ++unit_iter;
> +    }
> +  }
> +
> +  // For all SU in the suLevelDU list, look if remaining DU in tmpDU is 
> within
> +  // scope of the SU
> +  std::list < unitNameAndState >::iterator su_iter;
> +  for (su_iter = suLevelDU.begin(); su_iter != suLevelDU.end(); ++su_iter) {
> +    for (unit_iter = tmpDU.begin(); unit_iter != tmpDU.end();) {
> +      if ((*unit_iter).name.find((*su_iter).name) != std::string::npos) {
> +        // The component was in the scope of the SU
> +        TRACE("[%s] is in scope of [%s], remove it from DU list",
> +              (*unit_iter).name.c_str(), (*su_iter).name.c_str());
> +        // Remove the Component and update iterator
> +        unit_iter = tmpDU.erase(unit_iter);
> +      } else {
> +        ++unit_iter;
> +      }
> +    }
> +  }
> +
> +  newStep->addDeactivationUnits(nodeLevelDU);  // Add the node level DU
> +  newStep->addDeactivationUnits(suLevelDU);  // Add the SU level DU
> +  newStep->addDeactivationUnits(tmpDU);  // Add the comp level DU
> +
> +  // Rolling and forModify are symetric, add the node level DU
> +  newStep->addActivationUnits(nodeLevelDU);
> +  // Rolling and forModify are symetric, Add the SU level DU
> +  newStep->addActivationUnits(suLevelDU);
> +  // Rolling and forModify are symetric, Add the comp level DU
> +  newStep->addActivationUnits(tmpDU);
> +
> +  TRACE_LEAVE();
> +  return true;
> +}
> +
> +} // namespace execctrl
> diff --git a/osaf/services/saf/smfsv/smfd/SmfExecControl.h 
> b/osaf/services/saf/smfsv/smfd/SmfExecControl.h
> new file mode 100644
> --- /dev/null
> +++ b/osaf/services/saf/smfsv/smfd/SmfExecControl.h
> @@ -0,0 +1,59 @@
> +/*
> + *
> + * (C) Copyright 2016 The OpenSAF Foundation
> + *
> + * This program is distributed in the hope that it will be useful, but
> + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
> + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed
> + * under the GNU Lesser General Public License Version 2.1, February 1999.
> + * The complete license can be accessed from the following location:
> + * http://opensource.org/licenses/lgpl-license.php
> + * See the Copying file included with the OpenSAF distribution for full
> + * licensing terms.
> + *
> + * Author(s): Ericsson AB
> + *
> + */
> +#ifndef OPENSAF_STAGING_OSAF_SERVICES_SAF_SMFSV_SMFD_SMFEXECCONTROL_H_
> +#define OPENSAF_STAGING_OSAF_SERVICES_SAF_SMFSV_SMFD_SMFEXECCONTROL_H_
> +
> +#include <vector>
> +#include <string>
> +#include "base/macros.h"
> +#include "SmfUpgradeStep.hh"
> +#include "SmfUpgradeAction.hh"
> +#include "saAis.h"
> +
> +/*
> + * These set of functions enables the balanced mode feature. This mode 
> changes
> + * the execution of rolling procedures to be merged into one or several 
> single
> + * steps that are spread out across the cluster nodes. This feature is used 
> to
> + * give a faster upgrade time compared to rolling one node at a time, 
> possibly
> + * several times for each node. By splitting the procedures it into several
> + * single steps across the nodes a total service outage can be avoided.
> + */
> +namespace execctrl {
> +  /*
> +   * Creates empty procedures with the configuration provided in the
> +   * ExecControl object. The Campaign object need these empty procedures to
> +   * spawn procedure threads. Once started the steps will be built and added 
> to
> +   * the procedure.
> +   */
> +  bool createBalancedProcs();
> +  /*
> +   * Create the merged step for a balanced procedure. This merged step is 
> based
> +   * on steps from original procedures matching the balanced group.
> +   */
> +  bool createStepForBalancedProc(SmfUpgradeProcedure* procedure);
> +  /*
> +   * Check if an original procedure will have all its steps merged to 
> balanced
> +   * procedures. Set the step that will be merged to completed state.
> +   *
> +   * Return: true, if all steps will be merged
> +   */
> +  bool allStepsMerged(SmfUpgradeProcedure* procedure);
> +
> +  void trace(const std::string& message);
> +} // namespace execctrl
> +
> +#endif  // OPENSAF_STAGING_OSAF_SERVICES_SAF_SMFSV_SMFD_SMFEXECCONTROL_H_
> diff --git a/osaf/services/saf/smfsv/smfd/SmfProcState.cc 
> b/osaf/services/saf/smfsv/smfd/SmfProcState.cc
> --- a/osaf/services/saf/smfsv/smfd/SmfProcState.cc
> +++ b/osaf/services/saf/smfsv/smfd/SmfProcState.cc
> @@ -36,6 +36,7 @@
>   #include "SmfRollback.hh"
>   #include "SmfUtils.hh"
>   #include "smfd.h"
> +#include "SmfExecControl.h"
>   
>   /* ========================================================================
>    *   DEFINITIONS
> @@ -241,8 +242,9 @@ SmfProcStateInitial::executeInit(SmfUpgr
>   {
>       TRACE_ENTER();
>       LOG_NO("PROC: Start procedure init actions");
> -        if 
> (SmfCampaignThread::instance()->campaign()->getUpgradeCampaign()->getProcExecutionMode()
> -         == SMF_MERGE_TO_SINGLE_STEP) {
> +        auto camp = SmfCampaignThread::instance()->campaign();
> +        int procExecMode = 
> camp->getUpgradeCampaign()->getProcExecutionMode();
> +        if (procExecMode == SMF_MERGE_TO_SINGLE_STEP) {
>                   LOG_NO("SmfProcStateInitial::executeInit, Merge procedures 
> into single step");
>                   if( !i_proc->mergeStepIntoSingleStep(i_proc)) {
>                           changeState(i_proc, 
> SmfProcStateExecFailed::instance());
> @@ -250,6 +252,26 @@ SmfProcStateInitial::executeInit(SmfUpgr
>                           TRACE_LEAVE();
>                           return SMF_PROC_FAILED;
>                   }
> +        } else if (procExecMode == SMF_BALANCED_MODE) {
> +                if (i_proc->getBalancedGroup().empty()) {
> +                        TRACE("SmfProcStateInitial::executeInit, Calculate 
> steps for original procedure");
> +                        if (!i_proc->calculateSteps()) {
> +                                LOG_NO("SmfProcStateExecuting::Step 
> calculation failed");
> +                                return SMF_PROC_FAILED;
> +                        }
> +                        // Some steps will be merged, mark them as completed
> +                        execctrl::allStepsMerged(i_proc);
> +                        execctrl::trace("after procStateInit 
> allStepsMerged");
> +                } else {
> +                        LOG_NO("SmfProcStateInitial::executeInit, create 
> step for balanced procedure");
> +                        if (!execctrl::createStepForBalancedProc(i_proc)) {
> +                                changeState(i_proc, 
> SmfProcStateExecFailed::instance());
> +                                LOG_NO("SmfProcStateExecuting::executeInit: 
> failed to create balanced steps");
> +                                TRACE_LEAVE();
> +                                return SMF_PROC_FAILED;
> +                        }
> +                        execctrl::trace("after procStateInit 
> createStepsForBalancedProc");
> +                }
>           } else {
>                   TRACE("SmfProcStateInitial::executeInit, Calculate steps");
>                   if( !i_proc->calculateSteps() ) {
> @@ -351,10 +373,12 @@ SmfProcStateExecuting::executeStep(SmfUp
>       std::vector < SmfUpgradeStep * >::const_iterator iter;
>           const std::vector < SmfUpgradeStep * >& procSteps = 
> i_proc->getProcSteps();
>           unsigned int execStepNo = procSteps.size();
> +        TRACE("total -execStepNo %d", execStepNo);
>   
>           for (iter = procSteps.begin(); iter != procSteps.end(); iter++) {
>                   SmfStepResultT stepResult;
>                   execStepNo--;
> +                TRACE("-execStepNo %d, dn: %s", execStepNo, 
> (*iter)->getDn().c_str());
>   
>                   /* Try executing the step */
>                   stepResult = (*iter)->execute();
> diff --git a/osaf/services/saf/smfsv/smfd/SmfUpgradeCampaign.cc 
> b/osaf/services/saf/smfsv/smfd/SmfUpgradeCampaign.cc
> --- a/osaf/services/saf/smfsv/smfd/SmfUpgradeCampaign.cc
> +++ b/osaf/services/saf/smfsv/smfd/SmfUpgradeCampaign.cc
> @@ -84,22 +84,34 @@ class SmfUpgradeProcedure;
>   // 
> ------------------------------------------------------------------------------
>   SmfUpgradeCampaign::~SmfUpgradeCampaign()
>   {
> -     TRACE_ENTER();
> +        TRACE_ENTER();
>           //Delete merged procedure first since it contain references to 
> other proc
>       //Check campaign state, if verify fails the campaign is still in state 
> initial
>       //and the merged procedure is not yet created.
> -     if ((getProcExecutionMode() == SMF_MERGE_TO_SINGLE_STEP) &&
> +        int procexec = getProcExecutionMode();
> +        if ((procexec == SMF_MERGE_TO_SINGLE_STEP || procexec == 
> SMF_BALANCED_MODE) &&
>           (m_state->getState() != SA_SMF_CMPG_INITIAL)) {
> -                delete(m_mergedProcedure);
> +                for (auto& it: m_procedure) {
> +                        delete it;
> +                }
> +                m_procedure.clear();
> +        }
> +        for (auto& it: m_originalProcedures) {
> +                delete it;
>           }
> +        m_originalProcedures.clear();
> +        TRACE_LEAVE();
> +}
>   
> -     std::vector < SmfUpgradeProcedure * >::iterator iter;
> -
> -     for (iter = m_procedure.begin(); iter != m_procedure.end(); ++iter) {
> -             delete(*iter);
> -     }
> -
> -     TRACE_LEAVE();
> +// 
> ------------------------------------------------------------------------------
> +// addMergedProcedure()
> +// 
> ------------------------------------------------------------------------------
> +void SmfUpgradeCampaign::addMergedProcedure(SmfUpgradeProcedure* procedure) {
> +        if (m_originalProcedures.size() == 0) {
> +                m_originalProcedures = m_procedure;
> +                m_procedure.clear();
> +        }
> +        m_procedure.push_back(procedure);
>   }
>   
>   // 
> ------------------------------------------------------------------------------
> diff --git a/osaf/services/saf/smfsv/smfd/SmfUpgradeCampaign.hh 
> b/osaf/services/saf/smfsv/smfd/SmfUpgradeCampaign.hh
> --- a/osaf/services/saf/smfsv/smfd/SmfUpgradeCampaign.hh
> +++ b/osaf/services/saf/smfsv/smfd/SmfUpgradeCampaign.hh
> @@ -400,23 +400,19 @@ void verify();
>           SaAisErrorT checkSmfRestartIndicator();
>   
>   ///
> -/// Purpose: get procedure list
> -/// @param   None.
> -/// @return  list of procedures.
> +/// Purpose: Get the unmodified procedures, no merged procedures included
>   ///
> -     const std::vector < SmfUpgradeProcedure * >& getProcedures() { return 
> m_procedure; }
> +     const std::vector<SmfUpgradeProcedure*>& getOriginalProcedures() { 
> return m_originalProcedures; };
>   
> -/// Purpose: set the merged procedure
> -/// @param   A SmfUpgradeProcedure *
> -/// @return  the procedure.
>   ///
> -     void setMergedProc(SmfUpgradeProcedure * proc) { m_mergedProcedure = 
> proc; }
> +/// Purpose: Add the merged procedure while saving the original
> +///
> +     void addMergedProcedure(SmfUpgradeProcedure* procedure);
>   
> -/// Purpose: get the merged procedure
> -/// @param   None.
> -/// @return  the procedure.
>   ///
> -     SmfUpgradeProcedure * getMergedProc() { return m_mergedProcedure; }
> +/// Purpose: Can be used with any procedure execution mode
> +///
> +     const std::vector<SmfUpgradeProcedure*>& getProcedures() { return 
> m_procedure; }
>   
>   /// Purpose: Set the procedure ecxecution mode
>   /// @param   The execution mode.
> @@ -482,8 +478,8 @@ private:
>           std::string m_configurationBase;
>       SmfCampaignInit m_campInit;
>       SmfCampaignWrapup m_campWrapup;
> -        std::vector < SmfUpgradeProcedure * >m_procedure;
> -        SmfUpgradeProcedure * m_mergedProcedure;
> +        std::vector<SmfUpgradeProcedure*> m_procedure;
> +        std::vector<SmfUpgradeProcedure*> m_originalProcedures;
>           SaTimeT m_waitToCommit;
>           SaTimeT m_waitToAllowNewCampaign;
>       int m_noOfExecutingProc;
> diff --git a/osaf/services/saf/smfsv/smfd/SmfUpgradeProcedure.cc 
> b/osaf/services/saf/smfsv/smfd/SmfUpgradeProcedure.cc
> --- a/osaf/services/saf/smfsv/smfd/SmfUpgradeProcedure.cc
> +++ b/osaf/services/saf/smfsv/smfd/SmfUpgradeProcedure.cc
> @@ -51,6 +51,7 @@
>   #include "SmfUpgradeAction.hh"
>   #include "SmfUtils.hh"
>   #include "smfd.h"
> +#include "SmfExecControl.h"
>   
>   // This static member variable is the object counter for upgrade procedures
>   unsigned long SmfUpgradeProcedure::s_procCounter = 1;
> @@ -146,6 +147,14 @@ SmfUpgradeProcedure::~SmfUpgradeProcedur
>                           delete(*stepit);
>                   }
>           }
> +        //} else {
> +        //        unsigned int execMode = 
> SmfCampaignThread::instance()->campaign()->getUpgradeCampaign()->getProcExecutionMode();
> +        //        if (execMode == SMF_BALANCED_MODE) {
> +        //                for (auto& step : m_procSteps) {
> +        //                        delete step;
> +        //                }
> +        //        }
> +        //}
>   
>       TRACE_LEAVE();
>   }
> @@ -1171,7 +1180,7 @@ SmfUpgradeProcedure::mergeStepIntoSingle
>           std::list < unitNameAndState > forAddRemoveDU;
>       std::list < unitNameAndState > tmpDU;
>           SmfUpgradeCampaign * camp = 
> SmfCampaignThread::instance()->campaign()->getUpgradeCampaign();
> -        const std::vector < SmfUpgradeProcedure * >& procedures = 
> camp->getProcedures();
> +        const std::vector < SmfUpgradeProcedure * >& procedures = 
> camp->getOriginalProcedures();
>           std::vector < SmfUpgradeProcedure * >::const_iterator proc_iter;
>           for (proc_iter = procedures.begin(); proc_iter != procedures.end(); 
> proc_iter++) {
>                   LOG_NO("Merging [%s] into a single step procedure", 
> (*proc_iter)->getName().c_str());
> @@ -2775,12 +2784,16 @@ SmfUpgradeProcedure::getImmSteps()
>               return SA_AIS_ERR_NOT_EXIST;
>       }
>   
> +        unsigned int execMode = 
> SmfCampaignThread::instance()->campaign()->getUpgradeCampaign()->getProcExecutionMode();
>       if (upgradeMethod->getUpgradeMethod() == SA_SMF_ROLLING) {
> -             TRACE("Rolling upgrade");
> -             rc = getImmStepsRolling();
> +                TRACE("Rolling upgrade");
> +                rc = getImmStepsRolling();
> +                if (execMode == SMF_BALANCED_MODE) {
> +                        // We get here after a SI SWAP, some steps will be 
> merged, mark them as completed
> +                        execctrl::allStepsMerged(this);
> +                }
>       } else if (upgradeMethod->getUpgradeMethod() == SA_SMF_SINGLE_STEP) {
> -                if 
> (SmfCampaignThread::instance()->campaign()->getUpgradeCampaign()->getProcExecutionMode()
> -                 == SMF_MERGE_TO_SINGLE_STEP) {  //This is a merged single 
> step
> +                if (execMode == SMF_MERGE_TO_SINGLE_STEP) {  //This is a 
> merged single step
>                           TRACE("Merged single step upgrade");
>                           rc = getImmStepsMergedSingleStep();
>                   } else { //This is a written normal single step
> @@ -3140,7 +3153,7 @@ SmfUpgradeProcedure::getImmStepsMergedSi
>               int initActionId = 1;
>               int wrapupActionId = 1;
>               SmfUpgradeCampaign * camp = 
> SmfCampaignThread::instance()->campaign()->getUpgradeCampaign();
> -             const std::vector < SmfUpgradeProcedure * >& procedures = 
> camp->getProcedures();
> +             const std::vector < SmfUpgradeProcedure * >& procedures = 
> camp->getOriginalProcedures();
>               std::vector < SmfUpgradeProcedure * >::const_iterator proc_iter;
>               for (proc_iter = procedures.begin(); proc_iter != 
> procedures.end(); proc_iter++) {
>                       
> LOG_NO("SmfUpgradeProcedure::getImmStepsMergedSingleStep: Fetch callbacks and 
> wrapup actions from [%s]",
> @@ -3415,7 +3428,7 @@ SmfUpgradeProcedure::bundleRefFromSsCamp
>       //----------------------------------------------------------------
>       //Read all the bundles to add/remove from the parsed camaign
>           SmfUpgradeCampaign * camp = 
> SmfCampaignThread::instance()->campaign()->getUpgradeCampaign();
> -     const std::vector < SmfUpgradeProcedure * >& procedures = 
> camp->getProcedures();
> +     const std::vector < SmfUpgradeProcedure * >& procedures = 
> camp->getOriginalProcedures();
>       std::vector < SmfUpgradeProcedure * >::const_iterator proc_iter;
>       std::list < SmfBundleRef > bundlesOldProcSS;
>       std::list < SmfBundleRef *> bundlesOldProcRO;
> diff --git a/osaf/services/saf/smfsv/smfd/SmfUpgradeProcedure.hh 
> b/osaf/services/saf/smfsv/smfd/SmfUpgradeProcedure.hh
> --- a/osaf/services/saf/smfsv/smfd/SmfUpgradeProcedure.hh
> +++ b/osaf/services/saf/smfsv/smfd/SmfUpgradeProcedure.hh
> @@ -551,7 +551,7 @@ class SmfUpgradeProcedure {
>   /// @param    -
>   /// @return   The list of upgrade steps.
>   ///
> -        const std::vector < SmfUpgradeStep * >& getProcSteps() { return 
> m_procSteps; }
> +        std::vector < SmfUpgradeStep * >& getProcSteps() { return 
> m_procSteps; }
>   
>   ///
>   /// Purpose:  Add the list of upgrade steps
> @@ -717,8 +717,6 @@ class SmfUpgradeProcedure {
>       friend class SmfProcState;
>       friend class SmfCampStateSuspendingExec;
>   
> - private:
> -
>   ///
>   /// Purpose:  Get iformation from AMF config in IMM about Components, SUs 
> and nodes needed for upgrade
>   /// @param    A reference to a std::multimap<std::string, objectInst>
> @@ -727,6 +725,19 @@ class SmfUpgradeProcedure {
>       bool getImmComponentInfo(std::multimap<std::string, objectInst> 
> &i_objects);
>   
>   ///
> +/// Purpose: When merging with SMF_BALANCED_MODE we need to keep track of 
> which balanced group procedures belong to.
> +///
> +        const std::vector<std::string>& getBalancedGroup() { return 
> m_balancedGroup; }
> +
> +///
> +/// Purpose: When merging with SMF_BALANCED_MODE we need to keep track of 
> which balanced group procedures belong to.
> +///
> +        void setBalancedGroup(std::vector<std::string> group) { 
> m_balancedGroup = group; }
> +
> + private:
> +
> +
> +///
>   /// Purpose:  Change the procedure stste. If i_onlyInternalState == false, 
> the IMM procedure object is updated and
>   ///           a state change event is sent
>   /// @param    -
> @@ -765,6 +776,7 @@ class SmfUpgradeProcedure {
>           std::list < SmfCallback * >m_afterUnlock;   //Container of the 
> procedure callbacks to be invoked onstep, ataction
>       sem_t m_semaphore;
>           bool m_isMergedProcedure;
> +        std::vector<std::string> m_balancedGroup;
>   };
>   
>   //////////////////////////////////////////////////
> diff --git a/osaf/services/saf/smfsv/smfd/SmfUpgradeStep.hh 
> b/osaf/services/saf/smfsv/smfd/SmfUpgradeStep.hh
> --- a/osaf/services/saf/smfsv/smfd/SmfUpgradeStep.hh
> +++ b/osaf/services/saf/smfsv/smfd/SmfUpgradeStep.hh
> @@ -707,6 +707,12 @@ class SmfUpgradeStep {
>   /// @return   true on success else false
>   ///
>       bool readSmfClusterControllers();
> +///
> +/// Purpose:  Set step state
> +/// @param    The state
> +/// @return   None
> +///
> +     void setStepState(SaSmfStepStateT i_state);
>   
>           friend class SmfStepState;
>   
> @@ -761,13 +767,6 @@ class SmfUpgradeStep {
>       void setImmStateAndSendNotification(SaSmfStepStateT i_state);
>   
>   ///
> -/// Purpose:  Set step state
> -/// @param    The state
> -/// @return   None
> -///
> -     void setStepState(SaSmfStepStateT i_state);
> -
> -///
>   /// Purpose: Disables copy constructor
>   ///
>        SmfUpgradeStep(const SmfUpgradeStep &);
> diff --git a/osaf/services/saf/smfsv/smfd/smfd.h 
> b/osaf/services/saf/smfsv/smfd/smfd.h
> --- a/osaf/services/saf/smfsv/smfd/smfd.h
> +++ b/osaf/services/saf/smfsv/smfd/smfd.h
> @@ -60,7 +60,7 @@ extern "C" {
>   /* SMF execution modes */
>   #define SMF_STANDARD_MODE 0
>   #define SMF_MERGE_TO_SINGLE_STEP 1
> -#define SMF_MERGE_TO_NODE_ROLLING 2
> +#define SMF_BALANCED_MODE 2
>   
>   /* ========================================================================
>    *   TYPE DEFINITIONS


------------------------------------------------------------------------------
_______________________________________________
Opensaf-devel mailing list
Opensaf-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/opensaf-devel

Reply via email to