Hi Minh,
Ack(code review only).
Thanks
-Nagu
-----Original Message-----
From: Minh Chau [mailto:[email protected]]
Sent: 16 October 2018 03:46
To: [email protected]; [email protected];
[email protected]
Cc: [email protected]; Minh Chau
Subject: [PATCH 1/2] amf: Add new susi fsm EXCESSIVE state to handle
excessive assignment due to splitbrain V2 [#2929]
Once splitbrain happens, we have multiple partitions, in which AMF will
continue
assignments to the spare SUs in each partitions. When network merge, these
partitions
join into one cluster and the assignments of SU become excessive.
This patch adds a new susi fsm EXCESSIVE state, which is marked for the
excessive
assignments that AMF detects after multiple partitions join.
For 2N SG: Any excessive assignment exists, the SU that has 2N assignment
has its hosting node reboot
For NWay Active, NoRed: Remove the excessive assignment only.
For NpM, Nway: not supported
---
src/amf/amfd/cluster.cc | 5 ++++
src/amf/amfd/sg.cc | 33 +++++++++++++++++++++
src/amf/amfd/sg.h | 9 +++++-
src/amf/amfd/sg_2n_fsm.cc | 12 ++++++--
src/amf/amfd/sg_nored_fsm.cc | 19 ++++++++++++
src/amf/amfd/sg_nwayact_fsm.cc | 19 ++++++++++++
src/amf/amfd/sgproc.cc | 34 +++++++++++++---------
src/amf/amfd/si.cc | 11 +++++++
src/amf/amfd/si.h | 1 +
src/amf/amfd/siass.cc | 65
++++++++++++++++++++++++++++++++++--------
src/amf/amfd/susi.h | 5 +++-
11 files changed, 184 insertions(+), 29 deletions(-)
diff --git a/src/amf/amfd/cluster.cc b/src/amf/amfd/cluster.cc
index 83fd47d..07d9b5a 100644
--- a/src/amf/amfd/cluster.cc
+++ b/src/amf/amfd/cluster.cc
@@ -109,6 +109,11 @@ void avd_cluster_tmr_init_evh(AVD_CL_CB *cb, AVD_EVT
*evt) {
continue;
}
+ if (i_sg->any_assignment_excessive()) {
+ i_sg->failover_excessive_assignment();
+ continue;
+ }
+
while (i_sg->any_assignment_absent()) {
// failover with ABSENT SUSI, which had already been removed during
// headless, until all ABSENT SUSI(s) are failovered successfully
diff --git a/src/amf/amfd/sg.cc b/src/amf/amfd/sg.cc
index f973e3a..fa011ec 100644
--- a/src/amf/amfd/sg.cc
+++ b/src/amf/amfd/sg.cc
@@ -2332,6 +2332,39 @@ bool AVD_SG::any_assignment_absent() {
return pending;
}
+bool AVD_SG::any_assignment_excessive() {
+ bool pending = false;
+ TRACE_ENTER2("SG:'%s'", name.c_str());
+ for (const auto &su : list_of_su) {
+ if (su->any_susi_fsm_in(AVD_SU_SI_STATE_EXCESSIVE)) {
+ pending = true;
+ break;
+ }
+ }
+ TRACE_LEAVE();
+ return pending;
+}
+
+/*
+ * Going through all SU of this SG, if any SU has over assigned,
+ * reboot the node that hosts the SU.
+ */
+void AVD_SG::failover_excessive_assignment() {
+ TRACE_ENTER2("SG:'%s'", name.c_str());
+ for (const auto &su : list_of_su) {
+ if (su->list_of_susi != nullptr) {
+ if (su->saAmfSuReadinessState == SA_AMF_READINESS_IN_SERVICE) {
+ LOG_EM("Duplicated assignment SU '%s'", su->name.c_str());
+ LOG_EM("Sending node reboot order to '%s'",
+ su->su_on_node->name.c_str());
+ su->set_readiness_state(SA_AMF_READINESS_OUT_OF_SERVICE);
+ avd_d2n_reboot_snd(su->su_on_node);
+ }
+ }
+ }
+ TRACE_LEAVE();
+}
+
bool AVD_SG::any_assignment_in_progress() {
bool pending = false;
TRACE_ENTER2("SG:'%s'", name.c_str());
diff --git a/src/amf/amfd/sg.h b/src/amf/amfd/sg.h
index 81595a2..9d2b7c4 100644
--- a/src/amf/amfd/sg.h
+++ b/src/amf/amfd/sg.h
@@ -431,10 +431,16 @@ class AVD_SG {
* @return
*/
bool is_sg_serviceable_outside_ng(const AVD_AMF_NG *ng);
+ /*
+ * Failover the excessive assignment
+ */
+ virtual void failover_excessive_assignment();
+
SaAisErrorT check_sg_stability();
bool any_assignment_in_progress();
bool any_assignment_absent();
bool any_assignment_assigned();
+ bool any_assignment_excessive();
void failover_absent_assignment();
bool ng_using_saAmfSGAdminState;
bool headless_validation;
@@ -517,7 +523,7 @@ class SG_NORED : public AVD_SG {
struct avd_su_si_rel_tag *susi, AVSV_SUSI_ACT act,
SaAmfHAStateT state);
void ng_admin(AVD_SU *su, AVD_AMF_NG *ng);
-
+ void failover_excessive_assignment();
private:
AVD_SU *assign_sis_to_sus();
};
@@ -580,6 +586,7 @@ class SG_NACV : public AVD_SG {
struct avd_su_si_rel_tag *susi, AVSV_SUSI_ACT act,
SaAmfHAStateT state);
void ng_admin(AVD_SU *su, AVD_AMF_NG *ng);
+ void failover_excessive_assignment();
};
/**
diff --git a/src/amf/amfd/sg_2n_fsm.cc b/src/amf/amfd/sg_2n_fsm.cc
index af8a4cc..f919291 100644
--- a/src/amf/amfd/sg_2n_fsm.cc
+++ b/src/amf/amfd/sg_2n_fsm.cc
@@ -78,6 +78,7 @@ AVD_SU_SI_STATE avd_su_fsm_state_determine(AVD_SU *su) {
bool assigning_flag = false, assigned_flag = false, modify_flag = false,
unassingned_flag = false;
bool absent_flag = false;
+ bool excessive_flag = false;
AVD_SU_SI_STATE fsm_state = AVD_SU_SI_STATE_ABSENT;
TRACE_ENTER2("SU '%s'", su->name.c_str());
@@ -109,6 +110,10 @@ AVD_SU_SI_STATE avd_su_fsm_state_determine(AVD_SU *su)
{
absent_flag = true;
TRACE("Absent su'%s', si'%s'", temp_susi->su->name.c_str(),
temp_susi->si->name.c_str());
+ } else if (AVD_SU_SI_STATE_EXCESSIVE == temp_susi->fsm) {
+ excessive_flag = true;
+ TRACE("Excessive su'%s', si'%s'", temp_susi->su->name.c_str(),
+ temp_susi->si->name.c_str());
} else {
osafassert(0);
}
@@ -116,11 +121,14 @@ AVD_SU_SI_STATE avd_su_fsm_state_determine(AVD_SU *su)
{
}
TRACE(
- "assigning_flag'%u', unassingned_flag'%u', assigned_flag'%u',
modify_flag'%u', absent_flag'%u'",
+ "assigning_flag'%u', unassingned_flag'%u', assigned_flag'%u',"
+ "modify_flag'%u', absent_flag'%u', excessive_flag'%u'",
assigning_flag, unassingned_flag, assigned_flag, modify_flag,
- absent_flag);
+ absent_flag, excessive_flag);
if (absent_flag == true) {
fsm_state = AVD_SU_SI_STATE_ABSENT;
+ } if (excessive_flag == true) {
+ fsm_state = AVD_SU_SI_STATE_EXCESSIVE;
} else if (true == modify_flag) {
/* Rule 1. => If any one of the SUSI is Mod, then SU will be said to be
modified. The other SUSI can be in assigning/assigned state in
diff --git a/src/amf/amfd/sg_nored_fsm.cc b/src/amf/amfd/sg_nored_fsm.cc
index 04df6a6..0f13874 100644
--- a/src/amf/amfd/sg_nored_fsm.cc
+++ b/src/amf/amfd/sg_nored_fsm.cc
@@ -1234,4 +1234,23 @@ void SG_NORED::ng_admin(AVD_SU *su, AVD_AMF_NG *ng) {
return;
}
+/*
+ * Failover the excessive assignment by gracefully removing it
+ */
+void SG_NORED::failover_excessive_assignment() {
+ TRACE_ENTER2("SG:'%s'", name.c_str());
+ for (const auto &su : list_of_su) {
+ if (su->any_susi_fsm_in(AVD_SU_SI_STATE_EXCESSIVE) &&
+ su->saAmfSuReadinessState == SA_AMF_READINESS_IN_SERVICE) {
+ LOG_WA("SU '%s' has been excessively assigned, remove its
assignment",
+ su->name.c_str());
+ // only call su_fault once at a time
+ su->set_readiness_state(SA_AMF_READINESS_OUT_OF_SERVICE);
+ su_fault(avd_cb, su);
+ break;
+ }
+ }
+ TRACE_LEAVE();
+}
+
SG_NORED::~SG_NORED() {}
diff --git a/src/amf/amfd/sg_nwayact_fsm.cc b/src/amf/amfd/sg_nwayact_fsm.cc
index 68b76f1..ddc853b 100644
--- a/src/amf/amfd/sg_nwayact_fsm.cc
+++ b/src/amf/amfd/sg_nwayact_fsm.cc
@@ -2137,4 +2137,23 @@ void SG_NACV::ng_admin(AVD_SU *su, AVD_AMF_NG *ng) {
return;
}
+/*
+ * Failover the excessive assignment by gracefully removing it
+ */
+void SG_NACV::failover_excessive_assignment() {
+ TRACE_ENTER2("SG:'%s'", name.c_str());
+ for (const auto &su : list_of_su) {
+ if (su->any_susi_fsm_in(AVD_SU_SI_STATE_EXCESSIVE) &&
+ su->saAmfSuReadinessState == SA_AMF_READINESS_IN_SERVICE) {
+ LOG_WA("SU '%s' has been excessively assigned, remove its
assignment",
+ su->name.c_str());
+ // only call su_fault once at a time for
+ su->set_readiness_state(SA_AMF_READINESS_OUT_OF_SERVICE);
+ su_fault(avd_cb, su);
+ break;
+ }
+ }
+ TRACE_LEAVE();
+}
+
SG_NACV::~SG_NACV() {}
diff --git a/src/amf/amfd/sgproc.cc b/src/amf/amfd/sgproc.cc
index 2bee875..8513132 100644
--- a/src/amf/amfd/sgproc.cc
+++ b/src/amf/amfd/sgproc.cc
@@ -1751,7 +1751,9 @@ void avd_su_si_assign_evh(AVD_CL_CB *cb, AVD_EVT *evt)
{
done:
if (su != nullptr) {
if (su->sg_of_su->sg_ncs_spec == false) {
- if (su->sg_of_su->any_assignment_absent() == true) {
+ if (su->sg_of_su->any_assignment_excessive() == true) {
+ su->sg_of_su->failover_excessive_assignment();
+ } else if (su->sg_of_su->any_assignment_absent() == true) {
su->sg_of_su->failover_absent_assignment();
}
} else {
@@ -2293,7 +2295,9 @@ void avd_node_down_appl_susi_failover(AVD_CL_CB *cb,
AVD_AVND *avnd) {
/* Free all the SU SI assignments*/
i_su->delete_all_susis();
- if (i_su->sg_of_su->any_assignment_absent() == true) {
+ if (i_su->sg_of_su->any_assignment_excessive() == true) {
+ i_su->sg_of_su->failover_excessive_assignment();
+ } else if (i_su->sg_of_su->any_assignment_absent() == true) {
i_su->sg_of_su->failover_absent_assignment();
}
/* Since a SU has gone out of service relook at the SG to
@@ -2446,7 +2450,7 @@ uint32_t avd_sg_su_si_mod_snd(AVD_CL_CB *cb, AVD_SU
*su, SaAmfHAStateT state) {
uint32_t rc = NCSCC_RC_FAILURE;
AVD_SU_SI_REL *i_susi;
SaAmfHAStateT old_ha_state = SA_AMF_HA_ACTIVE;
- AVD_SU_SI_STATE old_state = AVD_SU_SI_STATE_ASGN;
+ AVD_SU_SI_STATE old_fsm_state = AVD_SU_SI_STATE_ASGN;
TRACE_ENTER2("'%s', state %u", su->name.c_str(), state);
@@ -2471,13 +2475,15 @@ uint32_t avd_sg_su_si_mod_snd(AVD_CL_CB *cb, AVD_SU
*su, SaAmfHAStateT state) {
}
old_ha_state = i_susi->state;
- old_state = i_susi->fsm;
+ old_fsm_state = i_susi->fsm;
i_susi->state = state;
avd_susi_update_fsm(i_susi, AVD_SU_SI_STATE_MODIFY);
m_AVSV_SEND_CKPT_UPDT_ASYNC_UPDT(cb, i_susi, AVSV_CKPT_AVD_SI_ASS);
- avd_susi_update_assignment_counters(i_susi, AVSV_SUSI_ACT_MOD,
old_ha_state,
- state);
+ if (old_fsm_state != AVD_SU_SI_STATE_EXCESSIVE) {
+ avd_susi_update_assignment_counters(i_susi, AVSV_SUSI_ACT_MOD,
+ old_ha_state, state);
+ }
i_susi = i_susi->su_next;
}
@@ -2497,7 +2503,7 @@ uint32_t avd_sg_su_si_mod_snd(AVD_CL_CB *cb, AVD_SU
*su, SaAmfHAStateT state) {
}
i_susi->state = old_ha_state;
- avd_susi_update_fsm(i_susi, old_state);
+ avd_susi_update_fsm(i_susi, old_fsm_state);
m_AVSV_SEND_CKPT_UPDT_ASYNC_UPDT(cb, i_susi, AVSV_CKPT_AVD_SI_ASS);
i_susi = i_susi->su_next;
}
@@ -2562,7 +2568,7 @@ uint32_t
avd_sg_susi_mod_snd_honouring_si_dependency(AVD_SU *su,
uint32_t avd_sg_su_si_del_snd(AVD_CL_CB *cb, AVD_SU *su) {
uint32_t rc = NCSCC_RC_FAILURE;
AVD_SU_SI_REL *i_susi;
- AVD_SU_SI_STATE old_state = AVD_SU_SI_STATE_ASGN;
+ AVD_SU_SI_STATE old_fsm_state = AVD_SU_SI_STATE_ASGN;
TRACE_ENTER2("'%s'", su->name.c_str());
@@ -2575,14 +2581,16 @@ uint32_t avd_sg_su_si_del_snd(AVD_CL_CB *cb, AVD_SU
*su) {
/* change the state for all assignments to the specified state. */
i_susi = su->list_of_susi;
while (i_susi != AVD_SU_SI_REL_NULL) {
- old_state = i_susi->fsm;
+ old_fsm_state = i_susi->fsm;
if (i_susi->fsm != AVD_SU_SI_STATE_UNASGN) {
avd_susi_update_fsm(i_susi, AVD_SU_SI_STATE_UNASGN);
m_AVSV_SEND_CKPT_UPDT_ASYNC_UPDT(cb, i_susi, AVSV_CKPT_AVD_SI_ASS);
/* Update the assignment counters */
- avd_susi_update_assignment_counters(i_susi, AVSV_SUSI_ACT_DEL,
- static_cast<SaAmfHAStateT>(0),
- static_cast<SaAmfHAStateT>(0));
+ if (old_fsm_state != AVD_SU_SI_STATE_EXCESSIVE) {
+ avd_susi_update_assignment_counters(i_susi, AVSV_SUSI_ACT_DEL,
+ static_cast<SaAmfHAStateT>(0),
+ static_cast<SaAmfHAStateT>(0));
+ }
}
i_susi = i_susi->su_next;
}
@@ -2595,7 +2603,7 @@ uint32_t avd_sg_su_si_del_snd(AVD_CL_CB *cb, AVD_SU
*su) {
LOG_ER("%s: avd_snd_susi_msg failed, %s", __FUNCTION__,
su->name.c_str());
i_susi = su->list_of_susi;
while (i_susi != AVD_SU_SI_REL_NULL) {
- avd_susi_update_fsm(i_susi, old_state);
+ avd_susi_update_fsm(i_susi, old_fsm_state);
m_AVSV_SEND_CKPT_UPDT_ASYNC_UPDT(cb, i_susi, AVSV_CKPT_AVD_SI_ASS);
i_susi = i_susi->su_next;
}
diff --git a/src/amf/amfd/si.cc b/src/amf/amfd/si.cc
index cf026d7..074c012 100644
--- a/src/amf/amfd/si.cc
+++ b/src/amf/amfd/si.cc
@@ -1674,6 +1674,17 @@ uint32_t AVD_SI::count_sisu_with(SaAmfHAStateT ha) {
}
return count;
}
+/*
+ * @brief Count number of SUSI assignments that are assigned to this SI
+ * (regardless HA state)
+ * @return: number of SUSI assignment
+ */
+uint32_t AVD_SI::count_sisu() {
+ uint32_t count = 0;
+ for (AVD_SU_SI_REL *sisu = list_of_sisu; sisu != nullptr;
+ sisu = sisu->si_next) count++;
+ return count;
+}
/*
* @brief Update alarm_sent by new value of @alarm_state,
diff --git a/src/amf/amfd/si.h b/src/amf/amfd/si.h
index 45b37cc..5821ef3 100644
--- a/src/amf/amfd/si.h
+++ b/src/amf/amfd/si.h
@@ -153,6 +153,7 @@ class AVD_SI {
bool is_active() const;
SaAisErrorT si_swap_validate();
uint32_t count_sisu_with(SaAmfHAStateT ha);
+ uint32_t count_sisu();
bool is_all_sponsor_si_unassigned() const;
bool is_all_dependent_si_unassigned() const;
diff --git a/src/amf/amfd/siass.cc b/src/amf/amfd/siass.cc
index 267c55c..c3f04a6 100644
--- a/src/amf/amfd/siass.cc
+++ b/src/amf/amfd/siass.cc
@@ -258,10 +258,20 @@ void avd_susi_read_headless_cached_rta(AVD_CL_CB *cb)
{
// the last fsm state when AMFD was before headless. This needs
// AMFND to resend susi_resp message if CSI completes during
// headless period.
- susi->fsm = imm_susi_fsm;
+ if (imm_susi_fsm != AVD_SU_SI_STATE_BASE &&
+ imm_susi_fsm != AVD_SU_SI_STATE_ABSENT) {
+ susi->fsm = imm_susi_fsm;
+ }
+
#endif
+ // validate SUSI assignments that are over assigned
+ if (avd_susi_validate_excessive_assignment(susi) == true) {
+ susi->fsm = AVD_SU_SI_STATE_EXCESSIVE;
+ }
+
// Checkpoint to add this SUSI
m_AVSV_SEND_CKPT_UPDT_ASYNC_ADD(avd_cb, susi, AVSV_CKPT_AVD_SI_ASS);
+
// restore assignment counter
if (susi->fsm == AVD_SU_SI_STATE_ASGN ||
susi->fsm == AVD_SU_SI_STATE_ASGND ||
@@ -446,6 +456,34 @@ done:
present_susi->su->sg_of_su->headless_validation);
return present_susi->su->sg_of_su->headless_validation;
}
+
+/**
+ * Validate the excessively present assignment
+ * @param present_susi
+ * @return: true of excessively, false otherwise
+ */
+bool avd_susi_validate_excessive_assignment(AVD_SU_SI_REL *present_susi) {
+ bool too_many = false;
+ TRACE_ENTER();
+ AVD_SI *si = present_susi->si;
+ AVD_SU *su = present_susi->su;
+ SaAmfHAStateT ha_state = present_susi->state;
+ if (su->sg_of_su->sg_redundancy_model == SA_AMF_2N_REDUNDANCY_MODEL) {
+ if (si->count_sisu_with(ha_state) > 1 ||
+ si->count_sisu() > 2) too_many = true;
+ }
+ if (su->sg_of_su->sg_redundancy_model == SA_AMF_NO_REDUNDANCY_MODEL) {
+ if (si->curr_active_assignments() >= 1) too_many = true;
+ }
+ if (su->sg_of_su->sg_redundancy_model
+ == SA_AMF_N_WAY_ACTIVE_REDUNDANCY_MODEL) {
+ if (si->curr_active_assignments() >=
+ si->pref_active_assignments()) too_many = true;
+ }
+ // TODO: Check for NpM and NWay SG type
+ TRACE_LEAVE2("%u", too_many);
+ return too_many;
+}
/***************************************************************************
**
* Function: avd_susi_create
*
@@ -847,9 +885,10 @@ uint32_t avd_susi_mod_send(AVD_SU_SI_REL *susi,
SaAmfHAStateT ha_state) {
goto done;
}
/* Update the assignment counters */
- avd_susi_update_assignment_counters(susi, AVSV_SUSI_ACT_MOD, old_state,
- ha_state);
-
+ if (old_fsm_state != AVD_SU_SI_STATE_EXCESSIVE) {
+ avd_susi_update_assignment_counters(susi, AVSV_SUSI_ACT_MOD, old_state,
+ ha_state);
+ }
/* Following updations has to be done after getting response, will take
care
* subsequently */
m_AVSV_SEND_CKPT_UPDT_ASYNC_UPDT(avd_cb, susi, AVSV_CKPT_AVD_SI_ASS);
@@ -891,10 +930,11 @@ uint32_t avd_susi_del_send(AVD_SU_SI_REL *susi) {
goto done;
}
/* Update the assignment counters */
- avd_susi_update_assignment_counters(susi, AVSV_SUSI_ACT_DEL,
- static_cast<SaAmfHAStateT>(0),
- static_cast<SaAmfHAStateT>(0));
-
+ if (old_fsm_state != AVD_SU_SI_STATE_EXCESSIVE) {
+ avd_susi_update_assignment_counters(susi, AVSV_SUSI_ACT_DEL,
+ static_cast<SaAmfHAStateT>(0),
+ static_cast<SaAmfHAStateT>(0));
+ }
/* Checkpointing has to be done after getting response, will take care
* subsequently */
m_AVSV_SEND_CKPT_UPDT_ASYNC_UPDT(avd_cb, susi, AVSV_CKPT_AVD_SI_ASS);
@@ -922,10 +962,11 @@ void avd_susi_update_assignment_counters(AVD_SU_SI_REL
*susi,
SaAmfHAStateT new_ha_state) {
AVD_SU *su = susi->su;
- TRACE_ENTER2("SI:'%s', SU:'%s' action:%u current_ha_state:%u
new_ha_state:%u",
- susi->si->name.c_str(), susi->su->name.c_str(), action,
- current_ha_state, new_ha_state);
+ TRACE_ENTER2("SI:'%s', SU:'%s' action:%u current_ha_state:%u
new_ha_state:%u"
+ " fsm:%u", susi->si->name.c_str(), susi->su->name.c_str(),
+ action, current_ha_state, new_ha_state, susi->fsm);
+ if (susi->fsm == AVD_SU_SI_STATE_EXCESSIVE) goto done;
switch (action) {
case AVSV_SUSI_ACT_ASGN:
if (new_ha_state == SA_AMF_HA_ACTIVE) {
@@ -982,7 +1023,7 @@ void avd_susi_update_assignment_counters(AVD_SU_SI_REL
*susi,
LOG_ER("%s: invalid action %u", __FUNCTION__, action);
break;
}
-
+done:
TRACE_LEAVE();
}
/**
diff --git a/src/amf/amfd/susi.h b/src/amf/amfd/susi.h
index 5d925ba..dde4395 100644
--- a/src/amf/amfd/susi.h
+++ b/src/amf/amfd/susi.h
@@ -44,7 +44,8 @@ typedef enum {
AVD_SU_SI_STATE_ASGN,
AVD_SU_SI_STATE_ASGND,
AVD_SU_SI_STATE_UNASGN,
- AVD_SU_SI_STATE_MODIFY
+ AVD_SU_SI_STATE_MODIFY,
+ AVD_SU_SI_STATE_EXCESSIVE
} AVD_SU_SI_STATE;
/* Availability directors SU SI relationship structure(AVD_SU_SI_REL):
@@ -146,6 +147,8 @@ void avd_susi_update_fsm(AVD_SU_SI_REL *susi,
AVD_SU_SI_STATE new_fsm_state);
bool avd_susi_validate_present_assignment(AVD_SU_SI_REL *present_susi,
SaAmfHAStateT ha_fr_imm,
AVD_SU_SI_STATE fsm_fr_imm);
+bool avd_susi_validate_excessive_assignment(AVD_SU_SI_REL *present_susi);
+
bool avd_susi_validate_absent_assignment(AVD_SU *su, AVD_SI *si,
SaAmfHAStateT imm_ha_state, AVD_SU_SI_STATE imm_fsm_state);
void avd_susi_read_headless_cached_rta(AVD_CL_CB *cb);
--
2.7.4
_______________________________________________
Opensaf-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/opensaf-devel
