The totem specification requires us to "deliver a transitional
configuration of all old members transitioning to the new ring". The
current code implementation delivers a transitional configuration change
of all old members that are also in the new membership. Details matter
here.
This fixes bugzilla related to totem token timeout being set to large
values and then killing and starting corosync again resulting in odd
behavior from cpg services (and the rest of the corosync stack).
Keep in mind this is a work in progress. Honza has agreed to clean it
up as I am off on vacation for a few days.
Ignore the logging revert (trunk was temporarily broken).
Regards
-steve
Index: include/corosync/engine/coroapi.h
===================================================================
--- include/corosync/engine/coroapi.h (revision 2644)
+++ include/corosync/engine/coroapi.h (working copy)
@@ -136,8 +136,9 @@
#endif /* COROSYNC_FLOW_CONTROL_STATE */
enum cs_sync_mode {
- CS_SYNC_V1 = 0,
- CS_SYNC_V2 = 1
+ CS_SYNC_V1 = 0,
+ CS_SYNC_V2 = 1,
+ CS_SYNC_V1_APIV2 = 2
};
typedef enum {
Index: services/cpg.c
===================================================================
--- services/cpg.c (revision 2644)
+++ services/cpg.c (working copy)
@@ -240,7 +240,9 @@
static int cpg_exec_send_joinlist(void);
-static void cpg_sync_init (
+static void cpg_sync_init_v2 (
+ const unsigned int *trans_list,
+ size_t trans_list_entries,
const unsigned int *member_list,
size_t member_list_entries,
const struct memb_ring_id *ring_id);
@@ -329,8 +331,8 @@
.exec_dump_fn = NULL,
.exec_engine = cpg_exec_engine,
.exec_engine_count = sizeof (cpg_exec_engine) / sizeof (struct corosync_exec_handler),
- .sync_mode = CS_SYNC_V1,
- .sync_init = cpg_sync_init,
+ .sync_mode = CS_SYNC_V1_APIV2,
+ .sync_init = cpg_sync_init_v2,
.sync_process = cpg_sync_process,
.sync_activate = cpg_sync_activate,
.sync_abort = cpg_sync_abort
@@ -406,7 +408,9 @@
static struct req_exec_cpg_downlist g_req_exec_cpg_downlist;
-static void cpg_sync_init (
+static void cpg_sync_init_v2 (
+ const unsigned int *trans_list,
+ size_t trans_list_entries,
const unsigned int *member_list,
size_t member_list_entries,
const struct memb_ring_id *ring_id)
@@ -435,8 +439,8 @@
*/
for (i = 0; i < my_old_member_list_entries; i++) {
found = 0;
- for (j = 0; j < my_member_list_entries; j++) {
- if (my_old_member_list[i] == my_member_list[j]) {
+ for (j = 0; j < trans_list_entries; j++) {
+ if (my_old_member_list[i] == trans_list[j]) {
found = 1;
break;
}
Index: exec/evil.c
===================================================================
--- exec/evil.c (revision 2644)
+++ exec/evil.c (working copy)
@@ -117,16 +117,18 @@
unsigned int endian_conversion_required);
static struct sync_callbacks clm_sync_operations = {
+ .api_version = 1,
.name = "dummy CLM service",
- .sync_init = clm_sync_init,
+ .sync_init_api.sync_init_v1 = clm_sync_init,
.sync_process = clm_sync_process,
.sync_activate = clm_sync_activate,
.sync_abort = clm_sync_abort,
};
static struct sync_callbacks evt_sync_operations = {
+ .api_version = 1,
.name = "dummy EVT service",
- .sync_init = evt_sync_init,
+ .sync_init_api.sync_init_v1 = evt_sync_init,
.sync_process = evt_sync_process,
.sync_activate = evt_sync_activate,
.sync_abort = evt_sync_abort,
@@ -212,7 +214,7 @@
}
if (callbacks_init) {
- callbacks->sync_init = sync_dummy_init;
+ callbacks->sync_init_api.sync_init_v1 = sync_dummy_init;
callbacks->sync_process = sync_dummy_process;
callbacks->sync_activate = sync_dummy_activate;
callbacks->sync_abort = sync_dummy_abort;
Index: exec/syncv2.c
===================================================================
--- exec/syncv2.c (revision 2644)
+++ exec/syncv2.c (working copy)
@@ -202,7 +202,7 @@
if (res == -1) {
continue;
}
- if (sync_callbacks.sync_init == NULL) {
+ if (sync_callbacks.sync_init_api.sync_init_v1 == NULL) {
continue;
}
my_initial_service_list[my_initial_service_list_entries].state =
@@ -210,7 +210,7 @@
my_initial_service_list[my_initial_service_list_entries].service_id = i;
strcpy (my_initial_service_list[my_initial_service_list_entries].name,
sync_callbacks.name);
- my_initial_service_list[my_initial_service_list_entries].sync_init = sync_callbacks.sync_init;
+ my_initial_service_list[my_initial_service_list_entries].sync_init = sync_callbacks.sync_init_api.sync_init_v1;
my_initial_service_list[my_initial_service_list_entries].sync_process = sync_callbacks.sync_process;
my_initial_service_list[my_initial_service_list_entries].sync_abort = sync_callbacks.sync_abort;
my_initial_service_list[my_initial_service_list_entries].sync_activate = sync_callbacks.sync_activate;
Index: exec/coroparse.c
===================================================================
--- exec/coroparse.c (revision 2644)
+++ exec/coroparse.c (working copy)
@@ -115,7 +115,6 @@
char line[512];
int i;
char *loc;
- int ignore_line;
while (fgets (line, sizeof (line), fp)) {
if (strlen(line) > 0) {
@@ -134,20 +133,10 @@
break;
}
}
-
- ignore_line = 1;
- for (i = 0; i < strlen (line); i++) {
- if (line[i] != '\t' && line[i] != ' ') {
- if (line[i] != '#')
- ignore_line = 0;
-
- break;
- }
- }
/*
* Clear out comments and empty lines
*/
- if (ignore_line) {
+ if (line[0] == '#' || line[0] == '\0') {
continue;
}
Index: exec/mainconfig.c
===================================================================
--- exec/mainconfig.c (revision 2644)
+++ exec/mainconfig.c (working copy)
@@ -255,53 +255,6 @@
return (-1);
}
-static int corosync_main_config_log_destination_set (
- struct objdb_iface_ver0 *objdb,
- hdb_handle_t object_handle,
- const char *subsys,
- const char **error_string,
- const char *objdb_key,
- unsigned int mode_mask,
- char deprecated,
- const char *replacement)
-{
- static char formatted_error_reason[128];
- char *value;
- unsigned int mode;
-
- if (!objdb_get_string (objdb, object_handle, objdb_key, &value)) {
- if (deprecated) {
- log_printf(LOGSYS_LEVEL_WARNING,
- "Warning: the %s config paramater has been obsoleted."
- " See corosync.conf man page %s directive.",
- objdb_key, replacement);
- }
-
- if (strcmp (value, "yes") == 0) {
- mode |= mode_mask;
- if (logsys_config_mode_set(subsys, mode) < 0) {
- sprintf (formatted_error_reason, "unable to set mode %s", objdb_key);
- *error_string = formatted_error_reason;
- return -1;
- }
- } else
- if (strcmp (value, "no") == 0) {
- mode &= ~mode_mask;
- if (logsys_config_mode_set(subsys, mode) < 0) {
- sprintf (formatted_error_reason, "unable to unset mode %s", objdb_key);
- *error_string = formatted_error_reason;
- return -1;
- }
- } else {
- sprintf (formatted_error_reason, "unknown value for %s", objdb_key);
- *error_string = formatted_error_reason;
- return -1;
- }
- }
-
- return 0;
-}
-
static int corosync_main_config_set (
struct objdb_iface_ver0 *objdb,
hdb_handle_t object_handle,
@@ -334,22 +287,91 @@
goto parse_error;
}
- if (corosync_main_config_log_destination_set (objdb, object_handle, subsys, &error_reason,
- "to_logfile", LOGSYS_MODE_OUTPUT_FILE, 0, NULL) != 0)
- goto parse_error;
+ if (!objdb_get_string (objdb,object_handle, "to_file", &value)) {
- if (corosync_main_config_log_destination_set (objdb, object_handle, subsys, &error_reason,
- "to_stderr", LOGSYS_MODE_OUTPUT_STDERR, 0, NULL) != 0)
- goto parse_error;
+ log_printf(LOGSYS_LEVEL_WARNING,
+ "Warning: the to_file config paramater has been obsoleted."
+ " See corosync.conf man page to_logfile directive.");
- if (corosync_main_config_log_destination_set (objdb, object_handle, subsys, &error_reason,
- "to_syslog", LOGSYS_MODE_OUTPUT_SYSLOG, 0, NULL) != 0)
- goto parse_error;
+ if (strcmp (value, "yes") == 0) {
+ mode |= LOGSYS_MODE_OUTPUT_FILE;
+ if (logsys_config_mode_set(subsys, mode) < 0) {
+ error_reason = "unable to set mode to_file";
+ goto parse_error;
+ }
+ } else
+ if (strcmp (value, "no") == 0) {
+ mode &= ~LOGSYS_MODE_OUTPUT_FILE;
+ if (logsys_config_mode_set(subsys, mode) < 0) {
+ error_reason = "unable to unset mode to_file";
+ goto parse_error;
+ }
+ } else {
+ error_reason = "unknown value for to_file";
+ goto parse_error;
+ }
+ }
- if (corosync_main_config_log_destination_set (objdb, object_handle, subsys, &error_reason,
- "to_file", LOGSYS_MODE_OUTPUT_FILE, 1, "to_logfile") != 0)
- goto parse_error;
+ if (!objdb_get_string (objdb,object_handle, "to_logfile", &value)) {
+ if (strcmp (value, "yes") == 0) {
+ mode |= LOGSYS_MODE_OUTPUT_FILE;
+ if (logsys_config_mode_set(subsys, mode) < 0) {
+ error_reason = "unable to set mode to_logfile";
+ goto parse_error;
+ }
+ } else
+ if (strcmp (value, "no") == 0) {
+ mode &= ~LOGSYS_MODE_OUTPUT_FILE;
+ if (logsys_config_mode_set(subsys, mode) < 0) {
+ error_reason = "unable to unset mode to_logfile";
+ goto parse_error;
+ }
+ } else {
+ error_reason = "unknown value for to_logfile";
+ goto parse_error;
+ }
+ }
+ if (!objdb_get_string (objdb,object_handle, "to_syslog", &value)) {
+ if (strcmp (value, "yes") == 0) {
+ mode |= LOGSYS_MODE_OUTPUT_SYSLOG;
+ if (logsys_config_mode_set(subsys, mode) < 0) {
+ error_reason = "unable to set mode to_syslog";
+ goto parse_error;
+ }
+ } else
+ if (strcmp (value, "no") == 0) {
+ mode &= ~LOGSYS_MODE_OUTPUT_SYSLOG;
+ if (logsys_config_mode_set(subsys, mode) < 0) {
+ error_reason = "unable to unset mode to_syslog";
+ goto parse_error;
+ }
+ } else {
+ error_reason = "unknown value for to_syslog";
+ goto parse_error;
+ }
+ }
+
+ if (!objdb_get_string (objdb,object_handle, "to_stderr", &value)) {
+ if (strcmp (value, "yes") == 0) {
+ mode |= LOGSYS_MODE_OUTPUT_STDERR;
+ if (logsys_config_mode_set(subsys, mode) < 0) {
+ error_reason = "unable to set mode to_stderr";
+ goto parse_error;
+ }
+ } else
+ if (strcmp (value, "no") == 0) {
+ mode &= ~LOGSYS_MODE_OUTPUT_STDERR;
+ if (logsys_config_mode_set(subsys, mode) < 0) {
+ error_reason = "unable to unset mode to_stderr";
+ goto parse_error;
+ }
+ } else {
+ error_reason = "unknown value for to_syslog";
+ goto parse_error;
+ }
+ }
+
if (!objdb_get_string (objdb,object_handle, "syslog_facility", &value)) {
int syslog_facility;
Index: exec/sync.c
===================================================================
--- exec/sync.c (revision 2644)
+++ exec/sync.c (working copy)
@@ -99,8 +99,12 @@
static unsigned int my_member_list[PROCESSOR_COUNT_MAX];
+static unsigned int my_trans_list[PROCESSOR_COUNT_MAX];
+
static unsigned int my_member_list_entries;
+static unsigned int my_trans_list_entries;
+
static int sync_barrier_send (const struct memb_ring_id *ring_id);
static int sync_start_process (enum totem_callback_token_type type,
@@ -181,7 +185,14 @@
static void sync_service_init (struct memb_ring_id *ring_id)
{
- sync_callbacks.sync_init (my_member_list, my_member_list_entries, ring_id);
+ if (sync_callbacks.api_version == 1) {
+ sync_callbacks.sync_init_api.sync_init_v1 (my_member_list,
+ my_member_list_entries, ring_id);
+ } else {
+ sync_callbacks.sync_init_api.sync_init_v2 (my_trans_list,
+ my_trans_list_entries,
+ my_member_list, my_member_list_entries, ring_id);
+ }
totempg_callback_token_destroy (&sync_callback_token_handle);
/*
@@ -227,7 +238,7 @@
break;
}
sync_recovery_index += 1;
- if (sync_callbacks.sync_init) {
+ if (sync_callbacks.sync_init_api.sync_init_v1) {
break;
}
}
@@ -434,7 +445,7 @@
/*
* if sync service found, execute it
*/
- if (sync_processing && sync_callbacks.sync_init) {
+ if (sync_processing && sync_callbacks.sync_init_api.sync_init_v1) {
log_printf (LOGSYS_LEVEL_DEBUG,
"Synchronization actions starting for (%s)\n",
sync_callbacks.name);
@@ -460,6 +471,9 @@
sync_ring_id = ring_id;
if (configuration_type != TOTEM_CONFIGURATION_REGULAR) {
+ memcpy (my_trans_list, member_list, member_list_entries *
+ sizeof (unsigned int));
+ my_trans_list_entries = member_list_entries;
return;
}
memcpy (my_member_list, member_list, member_list_entries * sizeof (unsigned int));
Index: exec/totemsrp.c
===================================================================
--- exec/totemsrp.c (revision 2644)
+++ exec/totemsrp.c (working copy)
@@ -323,6 +323,8 @@
struct srp_addr my_memb_list[PROCESSOR_COUNT_MAX];
+ struct memb_ring_id my_new_memb_ring_id_list[PROCESSOR_COUNT_MAX];
+
struct srp_addr my_deliver_memb_list[PROCESSOR_COUNT_MAX];
struct srp_addr my_left_memb_list[PROCESSOR_COUNT_MAX];
@@ -479,8 +481,6 @@
unsigned int old_ring_state_high_seq_received;
- int ring_saved;
-
unsigned int my_last_seq;
struct timeval tv_old;
@@ -1299,10 +1299,15 @@
return;
}
-static void memb_set_and (
- struct srp_addr *set1, int set1_entries,
- struct srp_addr *set2, int set2_entries,
- struct srp_addr *and, int *and_entries)
+static void memb_set_and_with_ring_id (
+ struct srp_addr *set1,
+ struct memb_ring_id *set1_ring_ids,
+ int set1_entries,
+ struct srp_addr *set2,
+ int set2_entries,
+ struct memb_ring_id *old_ring_id,
+ struct srp_addr *and,
+ int *and_entries)
{
int i;
int j;
@@ -1313,7 +1318,9 @@
for (i = 0; i < set2_entries; i++) {
for (j = 0; j < set1_entries; j++) {
if (srp_addr_equal (&set1[j], &set2[i])) {
- found = 1;
+ if (memcmp (&set1_ring_ids[j], old_ring_id, sizeof (struct memb_ring_id)) == 0) {
+ found = 1;
+ }
break;
}
}
@@ -1393,20 +1400,6 @@
}
}
-static void ring_save (struct totemsrp_instance *instance)
-{
- if (instance->ring_saved == 0) {
- instance->ring_saved = 1;
- memcpy (&instance->my_old_ring_id, &instance->my_ring_id,
- sizeof (struct memb_ring_id));
- }
-}
-
-static void ring_reset (struct totemsrp_instance *instance)
-{
- instance->ring_saved = 0;
-}
-
static void ring_state_restore (struct totemsrp_instance *instance)
{
if (instance->old_ring_state_saved) {
@@ -1694,7 +1687,7 @@
memb_consensus_reset (instance);
old_ring_state_reset (instance);
- ring_reset (instance);
+
deliver_messages_from_recovery_to_regular (instance);
log_printf (instance->totemsrp_log_level_debug,
@@ -1794,6 +1787,14 @@
reset_pause_timeout (instance);
+ /*
+ * Save ring id information from this configuration to determine
+ * which processors are transitioning from old regular configuration
+ * in to new regular configuration on the next configuration change
+ */
+ memcpy (&instance->my_old_ring_id, &instance->my_ring_id,
+ sizeof (struct memb_ring_id));
+
return;
}
@@ -1864,8 +1865,6 @@
static void memb_state_commit_enter (
struct totemsrp_instance *instance)
{
- ring_save (instance);
-
old_ring_state_save (instance);
memb_state_commit_token_update (instance);
@@ -1943,19 +1942,34 @@
/*
* Build transitional configuration
*/
- memb_set_and (instance->my_new_memb_list, instance->my_new_memb_entries,
- instance->my_memb_list, instance->my_memb_entries,
- instance->my_trans_memb_list, &instance->my_trans_memb_entries);
+ for (i = 0; i < instance->my_new_memb_entries; i++) {
+ memcpy (&instance->my_new_memb_ring_id_list[i],
+ &memb_list[i].ring_id,
+ sizeof (struct memb_ring_id));
+ }
+ memb_set_and_with_ring_id (
+ instance->my_new_memb_list,
+ instance->my_new_memb_ring_id_list,
+ instance->my_new_memb_entries,
+ instance->my_memb_list,
+ instance->my_memb_entries,
+ &instance->my_old_ring_id,
+ instance->my_trans_memb_list,
+ &instance->my_trans_memb_entries);
+ for (i = 0; i < instance->my_trans_memb_entries; i++) {
+ log_printf (instance->totemsrp_log_level_notice,
+ "TRANS [%d] member %s:\n", i, totemip_print (&instance->my_trans_memb_list[i].addr[0]));
+ }
for (i = 0; i < instance->my_new_memb_entries; i++) {
- log_printf (instance->totemsrp_log_level_debug,
+ log_printf (instance->totemsrp_log_level_notice,
"position [%d] member %s:\n", i, totemip_print (&addr[i].addr[0]));
- log_printf (instance->totemsrp_log_level_debug,
+ log_printf (instance->totemsrp_log_level_notice,
"previous ring seq %lld rep %s\n",
memb_list[i].ring_id.seq,
totemip_print (&memb_list[i].ring_id.rep));
- log_printf (instance->totemsrp_log_level_debug,
+ log_printf (instance->totemsrp_log_level_notice,
"aru %x high delivered %x received flag %d\n",
memb_list[i].aru,
memb_list[i].high_delivered,
@@ -2684,7 +2698,6 @@
memcpy (&memb_list[instance->commit_token->memb_index].ring_id,
&instance->my_old_ring_id, sizeof (struct memb_ring_id));
- assert (!totemip_zero_check(&instance->my_old_ring_id.rep));
memb_list[instance->commit_token->memb_index].aru = instance->old_ring_state_aru;
/*
Index: exec/sync.h
===================================================================
--- exec/sync.h (revision 2644)
+++ exec/sync.h (working copy)
@@ -39,11 +39,25 @@
#include <corosync/totem/totempg.h>
#include "totemsrp.h"
-struct sync_callbacks {
- void (*sync_init) (
+union sync_init_api {
+ void (*sync_init_v1) (
const unsigned int *member_list,
size_t member_list_entries,
const struct memb_ring_id *ring_id);
+
+ void (*sync_init_v2) (
+ const unsigned int *trans_list,
+ size_t trans_list_entries,
+ const unsigned int *member_list,
+ size_t member_list_entries,
+ const struct memb_ring_id *ring_id);
+};
+struct sync_callbacks {
+ int api_version;
+ /*
+ * UGH
+ */
+ union sync_init_api sync_init_api;
int (*sync_process) (void);
void (*sync_activate) (void);
void (*sync_abort) (void);
Index: exec/main.c
===================================================================
--- exec/main.c (revision 2644)
+++ exec/main.c (working copy)
@@ -276,7 +276,7 @@
ais_service_index++) {
if (ais_service[ais_service_index] != NULL
- && ais_service[ais_service_index]->sync_mode == CS_SYNC_V1) {
+ && ((ais_service[ais_service_index]->sync_mode == CS_SYNC_V1) || (ais_service[ais_service_index]->sync_mode == CS_SYNC_V1_APIV2))) {
if (ais_service_index == sync_id) {
break;
}
@@ -290,7 +290,11 @@
return (res);
}
callbacks->name = ais_service[ais_service_index]->name;
- callbacks->sync_init = ais_service[ais_service_index]->sync_init;
+ callbacks->sync_init_api.sync_init_v1 = ais_service[ais_service_index]->sync_init;
+ callbacks->api_version = 1;
+ if (ais_service[ais_service_index]->sync_mode == CS_SYNC_V1_APIV2) {
+ callbacks->api_version = 2;
+ }
callbacks->sync_process = ais_service[ais_service_index]->sync_process;
callbacks->sync_activate = ais_service[ais_service_index]->sync_activate;
callbacks->sync_abort = ais_service[ais_service_index]->sync_abort;
@@ -319,7 +323,7 @@
}
callbacks->name = ais_service[service_id]->name;
- callbacks->sync_init = ais_service[service_id]->sync_init;
+ callbacks->sync_init_api.sync_init_v1 = ais_service[service_id]->sync_init;
callbacks->sync_process = ais_service[service_id]->sync_process;
callbacks->sync_activate = ais_service[service_id]->sync_activate;
callbacks->sync_abort = ais_service[service_id]->sync_abort;
Index: man/corosync.conf.5
===================================================================
--- man/corosync.conf.5 (revision 2644)
+++ man/corosync.conf.5 (working copy)
@@ -41,9 +41,11 @@
.SH DESCRIPTION
The corosync.conf instructs the corosync executive about various parameters
-needed to control the corosync executive. Empty lines and lines starting with
-# character are ignored. The configuration file consists of bracketed top level
-directives. The possible directive choices are:
+needed to control the corosync executive. The configuration file consists of
+bracketed top level directives. The possible directive choices are
+.IR "totem { } , logging { }. It is also possible to specify the top level
+parameter compatibility.
+ These directives are described below.
.TP
totem { }
@@ -57,12 +59,12 @@
.PP
.PP
-It is also possible to specify the top level parameter
-.B compatibility.
-This directive indicates the level of compatibility requested by the user. The
+The
+.B compatibility
+directive indicates the level of compatibility requested by the user. The
option whitetank can be specified to remain backward compatable with
openais-0.80.z. The option none can be specified to only be compatable
-with corosync-1.Y.Z. Extra processing during configuration changes is
+with corsoync-1.Y.Z. Extra processing during configuration changes is
required to remain backward compatable.
The default is whitetank. (backwards compatibility)
_______________________________________________
Openais mailing list
[email protected]
https://lists.linux-foundation.org/mailman/listinfo/openais
