osaf/services/saf/immsv/immnd/ImmModel.cc | 9 +++++--
osaf/services/saf/immsv/immnd/immnd_init.h | 2 +-
osaf/services/saf/immsv/immnd/immnd_proc.c | 36 +++++++++++++++++++++++++++--
3 files changed, 40 insertions(+), 7 deletions(-)
The sync which is about to start is blocked waiting for ccbs to
complete. The logic that is periodically invoked to check this
is modified so that it detects if the PBE has been restarted AND
there are CCBs in critical. In that case deadlock is proclaimed
between PBE and sync and the sync is aborted.
There is always a waiting period of four seconds between syncs,
both after regular sync completion and aborted sync. This is to
allow the imm service to "come up for air", that is process other
urgent jobs, such as allowing a dump iteration to set epoch and start.
Testing this fix requires a testpatch that injects delay in the PBE
for the commit processing for a ccb, such as "ccb 3". Plus also
a delay in a restarted PBE to aquire the dump iterator.
This allows the tester tto trigger the ccb (using immcfg) and then
while the ccb is blocked, to kill the immnd coord, which will also
cause the PBE to terminate.
The restarted PBE will not be too quick in obtaining the dump
iterator, to allow the sync to get started => deadlock.
Without the fix, the deadlock will last for 20 seconds and then
both PBE and sync will abort and restart. With the patch, the sync
should get aborted soon after start. Then wait for 4 seconds before
restarting, allowing the PBE to proceed obtaining the dump and epoch.
Test patch will be provided on request.
diff --git a/osaf/services/saf/immsv/immnd/ImmModel.cc
b/osaf/services/saf/immsv/immnd/ImmModel.cc
--- a/osaf/services/saf/immsv/immnd/ImmModel.cc
+++ b/osaf/services/saf/immsv/immnd/ImmModel.cc
@@ -768,13 +768,16 @@ immModel_getOldCriticalCcbs(IMMND_CB *cb
}
}
-SaBoolT
+unsigned int
immModel_pbeOiExists(IMMND_CB *cb)
{
SaUint32T pbeConn=0;
unsigned int pbeNode=0;
- return (ImmModel::instance(&cb->immModel)->getPbeOi(&pbeConn, &pbeNode,
false)) ?
- SA_TRUE : SA_FALSE;
+ if(ImmModel::instance(&cb->immModel)->getPbeOi(&pbeConn, &pbeNode, false))
{
+ return pbeNode;
+ }
+
+ return 0;
}
SaBoolT
diff --git a/osaf/services/saf/immsv/immnd/immnd_init.h
b/osaf/services/saf/immsv/immnd/immnd_init.h
--- a/osaf/services/saf/immsv/immnd/immnd_init.h
+++ b/osaf/services/saf/immsv/immnd/immnd_init.h
@@ -261,7 +261,7 @@ extern "C" {
void immModel_setLoader(IMMND_CB *cb, SaInt32T loaderPid);
- SaBoolT immModel_pbeOiExists(IMMND_CB *cb);
+ unsigned int immModel_pbeOiExists(IMMND_CB *cb);
struct immsv_attr_values_list *
immModel_specialApplierTrimCreate(IMMND_CB *cb, SaUint32T clientId,
struct ImmsvOmCcbObjectCreate *req);
diff --git a/osaf/services/saf/immsv/immnd/immnd_proc.c
b/osaf/services/saf/immsv/immnd/immnd_proc.c
--- a/osaf/services/saf/immsv/immnd/immnd_proc.c
+++ b/osaf/services/saf/immsv/immnd/immnd_proc.c
@@ -900,7 +900,15 @@ static void immnd_cleanTheHouse(IMMND_CB
/*TRACE_ENTER(); */
if((cb->mRim == SA_IMM_KEEP_REPOSITORY) && !(cb->mPbeVeteran)) {
- cb->mPbeVeteran = immModel_pbeOiExists(cb) &&
immModel_pbeIsInSync(cb, false);
+
+ /*
+ If we are coord then the PBE has to be LOCAL.
+ If we are not coord then the PBE has to be remote.
+ */
+ SaUint32T nodeId = immModel_pbeOiExists(cb);
+ cb->mPbeVeteran = nodeId && immModel_pbeIsInSync(cb, false) &&
+ (cb->mIsCoord) == (nodeId == cb->node_id);
+
if(cb->mPbeVeteran && cb->mCanBeCoord) {
LOG_NO("PBE-OI established on %s SC. Dumping
incrementally to file %s",
(cb->mIsCoord)?"this":"other", cb->mPbeFile);
@@ -1162,9 +1170,11 @@ void immnd_proc_global_abort_ccb(IMMND_C
}
-static SaBoolT immnd_ccbsTerminated(IMMND_CB *cb, SaUint32T duration)
+static SaBoolT immnd_ccbsTerminated(IMMND_CB *cb, SaUint32T duration, SaBoolT*
pbeImmndDeadlock)
{
osafassert(cb->mIsCoord);
+ osafassert(pbeImmndDeadlock);
+ (*pbeImmndDeadlock) = SA_FALSE;
if (cb->mPendSync) {
TRACE("ccbsTerminated false because cb->mPendSync is still
true");
return SA_FALSE;
@@ -1203,6 +1213,12 @@ static SaBoolT immnd_ccbsTerminated(IMMN
}
}
+ if(!ccbsTerminated && cb->mPbeFile && (cb->mRim ==
SA_IMM_KEEP_REPOSITORY) &&
+ !immModel_pbeOiExists(cb) && !immModel_pbeIsInSync(cb, true)) {
+ /* Started sync is blocked by critical ccbs while PBE is being
restarted, See (#556) */
+ (*pbeImmndDeadlock) = SA_TRUE;
+ }
+
if (duration % 5) {/* Wait 5 secs for non critical ccbs to terminate */
return SA_FALSE;
}
@@ -1470,6 +1486,7 @@ uint32_t immnd_proc_server(uint32_t *tim
time_t now = time(NULL);
osafassert(now > ((time_t) 0));
uint32_t jobDuration = (uint32_t) now - cb->mJobStart;
+ SaBoolT pbeImmndDeadlock=SA_FALSE;
if(!jobDuration) {++jobDuration;} /* Avoid jobDuraton of zero */
/*TRACE_ENTER(); */
@@ -1723,8 +1740,21 @@ uint32_t immnd_proc_server(uint32_t *tim
break;
case IMM_SERVER_SYNC_SERVER:
- if (!immnd_ccbsTerminated(cb, jobDuration)) {
+ if (!immnd_ccbsTerminated(cb, jobDuration, &pbeImmndDeadlock)) {
/*Phase 1 */
+ if(pbeImmndDeadlock) {
+ LOG_WA("Apparent deadlock detected between
IMMND sync and restarting PBE, "
+ "with Ccbs in critical. Aborting this
sync attempt");
+ immnd_abortSync(cb);
+ if(cb->syncPid > 0) {
+ LOG_WA("STOPPING sync process pid %u",
cb->syncPid);
+ kill(cb->syncPid, SIGTERM);
+ }
+ cb->mStep = 0;
+ cb->mJobStart = now;
+ cb->mState = IMM_SERVER_READY;
+ LOG_NO("SERVER STATE: IMM_SERVER_SYNC_SERVER
--> IMM SERVER READY");
+ }
if (!(cb->mStep % 60)) {
LOG_IN("Sync Phase-1, waiting for existing "
"Ccbs to terminate, seconds waited: %u",
jobDuration);
------------------------------------------------------------------------------
Learn the latest--Visual Studio 2012, SharePoint 2013, SQL 2012, more!
Discover the easy way to master current and previous Microsoft technologies
and advance your career. Get an incredible 1,500+ hours of step-by-step
tutorial videos with LearnDevNow. Subscribe today and save!
http://pubads.g.doubleclick.net/gampad/clk?id=58040911&iu=/4140/ostg.clktrk
_______________________________________________
Opensaf-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/opensaf-devel
