Steven Dake wrote:
> Chrissie,
>
> Ok so here are comments:
>
> logsys.c has merged changes for fixing the wthread code. I will merge
> those separately.
>
> There are many cases of the 2nd line not using 1 tab but instead using
> several tabs and spaces. Example: quorum.h:39
>
> I'd like to see the vsf quorum layer more separated from the sync layer
> (work for later). For example, I'd like sync to register a callback
> with the coroapi when quorum is obtained and act accordingly. I suppose
> this can be part of the 0.94 version's work of reworking the
> synchronization system.
>
> I'd like to see a callback that we can register in the coroapi for
> quorum changes rather then only polling a function. I think both are
> probably necessary.
>
> The way I had always seen this being implemented is an exec/quorum.c
> file which is responsible for loading the virtual synchrony quorum
> filters based upon config options. A quorum_init would be called from
> main.c instead of sync in this case. The VSF filters would then work
> based upon majority voting, the ykd algo, qdisk, or whatever else. The
> sync system would register a callback with the coroapi to know when it
> could safely synchronize a cluster. The library api's service engine
> would register a callback in the same way to notify users or to provide
> polled information.
>
> The patch looks suitable to merge minus the whitespace and logsys
> changes. I would like us to keep cooking the implementation if you are
> not opposed though.
OK this should address those issues, apart from possibly the whitespace
ones!
quorum is now an optional loadable module (though I've put it into the
defaults in services.c) and can load another module to do the quorum
work (eg YKD which I've made more compliant too). All the quorum code
has been removed from sync.c. quorum.c is simple a shim later for the
coroapi, the main module is in vsf_quorum.c
There are coroapi calls to query quorate status and also to get
notifications when it changes.
I've included the testquorum.lcrso module in this patch because I think
it's really helpful for testing, feel free to veto it though I'm not
THAT attached to it!
Chrissie
Index: include/corosync/engine/coroapi.h
===================================================================
--- include/corosync/engine/coroapi.h (revision 1692)
+++ include/corosync/engine/coroapi.h (working copy)
@@ -188,9 +188,27 @@
typedef void (*object_reload_notify_fn_t) (objdb_reload_notify_type_t, int flush,
void *priv_data_pt);
-
#endif /* OBJECT_PARENT_HANDLE_DEFINED */
+#ifndef QUORUM_H_DEFINED
+typedef void (*quorum_callback_fn_t) (int quorate, void *context);
+
+struct quorum_callin_functions
+{
+ int (*quorate) (void);
+ int (*register_callback) (quorum_callback_fn_t callback_fn, void *context);
+ int (*unregister_callback) (quorum_callback_fn_t callback_fn, void *context);
+};
+
+typedef void (*sync_callback_fn_t) (
+ unsigned int *view_list,
+ int view_list_entries,
+ int primary_designated,
+ struct memb_ring_id *ring_id);
+
+#endif /* QUORUM_H_DEFINED */
+
+
struct corosync_api_v1 {
/*
* Object and configuration APIs
@@ -496,6 +514,19 @@
char *service_name);
/*
+ * User plugin-callable functions for quorum
+ */
+ int (*quorum_is_quorate) (void);
+ int (*quorum_register_callback) (quorum_callback_fn_t callback_fn, void *context);
+ int (*quorum_unregister_callback) (quorum_callback_fn_t callback_fn, void *context);
+
+ /*
+ * This one is for the quorum management plugin's use
+ */
+ int (*quorum_initialize)(struct quorum_callin_functions *fns,
+ sync_callback_fn_t *sync_callback_fn);
+
+ /*
* Plugin loading and unloading
*/
int (*plugin_interface_reference) (
Index: include/corosync/engine/quorum.h
===================================================================
--- include/corosync/engine/quorum.h (revision 0)
+++ include/corosync/engine/quorum.h (revision 0)
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2008 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Christine Caulfield ([EMAIL PROTECTED])
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the Red Hat Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef QUORUM_H_DEFINED
+#define QUORUM_H_DEFINED
+
+typedef void (*quorum_set_quorate_fn_t) (unsigned int *view_list, int view_list_entries,
+ int quorate, struct memb_ring_id *);
+
+struct quorum_services_api_ver1 {
+ void (*init) (struct corosync_api_v1 *api, quorum_set_quorate_fn_t);
+};
+#endif /* QUORUM_H_DEFINED */
Index: include/corosync/quorum.h
===================================================================
--- include/corosync/quorum.h (revision 1692)
+++ include/corosync/quorum.h (working copy)
@@ -15,7 +15,7 @@
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
+ * - Neither the name of the Red Hat, Inc. nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
Index: services/testquorum.c
===================================================================
--- services/testquorum.c (revision 0)
+++ services/testquorum.c (revision 0)
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2008 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Christine Caulfield ([EMAIL PROTECTED])
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of Red Hat, Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <assert.h>
+#include <pwd.h>
+#include <grp.h>
+#include <sys/types.h>
+#include <sys/poll.h>
+#include <sys/uio.h>
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <signal.h>
+#include <sched.h>
+#include <time.h>
+
+#include <corosync/engine/logsys.h>
+#include <corosync/ipc_gen.h>
+#include <corosync/lcr/lcr_comp.h>
+#include <corosync/engine/coroapi.h>
+
+#include <corosync/engine/quorum.h>
+
+LOGSYS_DECLARE_SUBSYS ("TEST", LOG_INFO);
+
+static void test_init(struct corosync_api_v1 *api, quorum_set_quorate_fn_t report);
+
+/*
+ * lcrso object definition
+ */
+static struct quorum_services_api_ver1 test_quorum_iface_ver0 = {
+ .init = test_init
+};
+
+static struct lcr_iface corosync_test_quorum_ver0[1] = {
+ {
+ .name = "testquorum",
+ .version = 0,
+ .versions_replace = 0,
+ .versions_replace_count = 0,
+ .dependencies = 0,
+ .dependency_count = 0,
+ .constructor = NULL,
+ .destructor = NULL,
+ .interfaces = (void **)(void *)&test_quorum_iface_ver0,
+ },
+};
+
+static struct lcr_comp test_quorum_comp_ver0 = {
+ .iface_count = 1,
+ .ifaces = corosync_test_quorum_ver0
+};
+
+__attribute__ ((constructor)) static void test_quorum_comp_register (void) {
+ lcr_interfaces_set (&corosync_test_quorum_ver0[0], &test_quorum_iface_ver0);
+ lcr_component_register (&test_quorum_comp_ver0);
+}
+
+/* -------------------------------------------------- */
+
+static quorum_set_quorate_fn_t set_quorum;
+
+static void key_change_notify(object_change_type_t change_type,
+ unsigned int parent_object_handle,
+ unsigned int object_handle,
+ void *object_name_pt, int object_name_len,
+ void *key_name_pt, int key_len,
+ void *key_value_pt, int key_value_len,
+ void *priv_data_pt)
+{
+ unsigned int members[1];
+ struct memb_ring_id ring_id;
+
+ memset(&ring_id, 0, sizeof(ring_id));
+
+ /* If the 'quorum.quorate' key changes, then that changes quorum */
+ if (strncmp(key_name_pt, "quorate", key_len) == 0) {
+ set_quorum(members, 0, atoi(key_value_pt), &ring_id);
+ }
+}
+
+static void quorum_callback(int quorate, void *context)
+{
+ log_printf(LOG_LEVEL_DEBUG, "quorum callback: quorate = %d\n", quorate);
+}
+
+static void test_init(struct corosync_api_v1 *api,
+ quorum_set_quorate_fn_t report)
+{
+
+ unsigned int find_handle;
+ unsigned int quorum_handle = 0;
+
+ set_quorum = report;
+
+ /*
+ * Register for objdb changes on quorum { }
+ */
+ api->object_find_create(OBJECT_PARENT_HANDLE, "quorum", strlen("quorum"), &find_handle);
+ api->object_find_next(find_handle, &quorum_handle);
+ api->object_find_destroy(find_handle);
+
+ api->object_track_start(quorum_handle,
+ 1,
+ key_change_notify,
+ NULL, // object_create_notify
+ NULL, // object_destroy_notify
+ NULL, // object_reload_notify
+ NULL); // priv_data
+
+ /* Register for quorum changes too! */
+ api->quorum_register_callback(quorum_callback, NULL);
+}
Index: services/confdb.c
===================================================================
--- services/confdb.c (revision 1692)
+++ services/confdb.c (working copy)
@@ -212,6 +212,7 @@
.id = CONFDB_SERVICE,
.private_data_size = 0,
.flow_control = CS_LIB_FLOW_CONTROL_NOT_REQUIRED,
+ .allow_inquorate = CS_LIB_ALLOW_INQUORATE,
.lib_init_fn = confdb_lib_init_fn,
.lib_exit_fn = confdb_lib_exit_fn,
.lib_engine = confdb_lib_engine,
Index: services/Makefile
===================================================================
--- services/Makefile (revision 1692)
+++ services/Makefile (working copy)
@@ -55,7 +55,7 @@
override CFLAGS += -fPIC
-all: service_evs.lcrso service_cfg.lcrso service_cpg.lcrso service_confdb.lcrso service_pload.lcrso
+all: service_evs.lcrso service_cfg.lcrso service_cpg.lcrso service_confdb.lcrso service_pload.lcrso testquorum.lcrso
ifeq (${COROSYNC_COMPAT}, DARWIN)
@@ -91,6 +91,9 @@
service_pload.lcrso: pload.o
$(CC) -shared -Wl,-soname,service_pload.lcrso pload.o -o $@
+testquorum.lcrso: testquorum.o
+ $(CC) -shared -Wl,-soname,testquorum.lcrso testquorum.o -o $@
+
endif
clean:
@@ -113,3 +116,6 @@
cpg.o: cpg.c
$(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<
+
+testquorum.o: testquorum.c
+ $(CC) $(CFLAGS) $(CPPFLAGS) -c -o $@ $<
Index: exec/service.c
===================================================================
--- exec/service.c (revision 1692)
+++ exec/service.c (working copy)
@@ -82,6 +82,10 @@
.name = "corosync_pload",
.ver = 0,
},
+ {
+ .name = "corosync_quorum",
+ .ver = 0,
+ },
};
struct corosync_service_engine *ais_service[SERVICE_HANDLER_MAXIMUM_COUNT];
Index: exec/vsf.h
===================================================================
--- exec/vsf.h (revision 1692)
+++ exec/vsf.h (working copy)
@@ -34,12 +34,14 @@
#ifndef VSF_H_DEFINED
#define VSF_H_DEFINED
+struct corosync_api_v1;
struct corosync_vsf_iface_ver0 {
/*
* Executes a callback whenever component changes
*/
int (*init) (
+ struct corosync_api_v1 *api,
void (*primary_callback_fn) (
unsigned int *view_list,
int view_list_entries,
Index: exec/vsf_quorum.c
===================================================================
--- exec/vsf_quorum.c (revision 1692)
+++ exec/vsf_quorum.c (working copy)
@@ -15,7 +15,7 @@
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
+ * - Neither the name of Red Hat Inc. nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
@@ -60,11 +60,10 @@
#include <corosync/ipc_gen.h>
#include <corosync/ipc_quorum.h>
#include <corosync/lcr/lcr_comp.h>
+#include <corosync/lcr/lcr_ifact.h>
#include <corosync/engine/coroapi.h>
+#include <corosync/engine/quorum.h>
-#include "vsf.h"
-#include "quorum.h"
-
LOGSYS_DECLARE_SUBSYS ("QUORUM", LOG_INFO);
struct quorum_pd {
@@ -74,22 +73,31 @@
void *conn;
};
+struct internal_callback_pd {
+ struct list_head list;
+ quorum_callback_fn_t callback;
+ void *context;
+};
+
static void message_handler_req_lib_quorum_getquorate (void *conn, void *msg);
static void message_handler_req_lib_quorum_trackstart (void *conn, void *msg);
static void message_handler_req_lib_quorum_trackstop (void *conn, void *msg);
-static int send_quorum_notification(void *conn);
+static void send_library_notification(void *conn);
+static void send_internal_notification(void);
static int quorum_exec_init_fn (struct corosync_api_v1 *api);
static int quorum_lib_init_fn (void *conn);
static int quorum_lib_exit_fn (void *conn);
static int primary_designated = 0;
static struct corosync_api_v1 *corosync_api;
-static struct list_head trackers_list;
+static struct list_head lib_trackers_list;
+static struct list_head internal_trackers_list;
static struct memb_ring_id quorum_ring_id;
static int quorum_view_list_entries = 0;
static int quorum_view_list[PROCESSOR_COUNT_MAX];
+struct quorum_services_api_ver1 *quorum_iface = NULL;
-static void (*quorum_primary_callback_fn) (
+static void (*sync_primary_callback_fn) (
unsigned int *view_list,
int view_list_entries,
int primary_designated,
@@ -101,51 +109,29 @@
int quorum, struct memb_ring_id *ring_id)
{
primary_designated = quorum;
- memcpy(&quorum_ring_id, &ring_id, sizeof (quorum_ring_id));
+ if (primary_designated) {
+ log_printf (LOG_LEVEL_NOTICE, "This node is within the primary component and will provide service.\n");
+ } else {
+ log_printf (LOG_LEVEL_NOTICE, "This node is within the non-primary component and will NOT provide any services.\n");
+ }
+
+ memcpy(&quorum_ring_id, ring_id, sizeof (quorum_ring_id));
+
quorum_view_list_entries = view_list_entries;
memcpy(quorum_view_list, view_list, sizeof(unsigned int)*view_list_entries);
/* Tell sync() */
- quorum_primary_callback_fn(view_list, view_list_entries,
- primary_designated, &quorum_ring_id);
+ sync_primary_callback_fn(view_list, view_list_entries,
+ primary_designated, &quorum_ring_id);
+ /* Tell internal listeners */
+ send_internal_notification();
+
/* Tell IPC listeners */
- send_quorum_notification(NULL);
+ send_library_notification(NULL);
}
-static int quorum_init (
- void (*primary_callback_fn) (
- unsigned int *view_list,
- int view_list_entries,
- int primary_designated,
- struct memb_ring_id *ring_id))
-{
- quorum_primary_callback_fn = primary_callback_fn;
-
- return (0);
-}
-
-/*
- * Returns 1 if this processor is in the primary (has quorum)
- */
-static int quorum_primary (void)
-{
- return (primary_designated);
-}
-
-/*
- * lcrso object definition
- */
-static struct corosync_vsf_iface_ver0 vsf_quorum_iface_ver0 = {
- .init = quorum_init,
- .primary = quorum_primary
-};
-
-static struct quorum_services_api_ver1 quorum_service_api_v1 = {
- .quorum_api_set_quorum = quorum_api_set_quorum
-};
-
static struct corosync_lib_handler quorum_lib_service[] =
{
{ /* 0 */
@@ -181,30 +167,8 @@
.lib_engine_count = sizeof (quorum_lib_service) / sizeof (struct corosync_lib_handler),
};
-static struct lcr_iface corosync_vsf_quorum_ver0[3] = {
- { /* the VSF handler */
- .name = "corosync_vsf_quorum",
- .version = 0,
- .versions_replace = 0,
- .versions_replace_count = 0,
- .dependencies = 0,
- .dependency_count = 0,
- .constructor = NULL,
- .destructor = NULL,
- .interfaces = (void **)(void *)&vsf_quorum_iface_ver0,
- },
- { /* API for quorum users to call */
- .name = "corosync_quorum_api",
- .version = 0,
- .versions_replace = 0,
- .versions_replace_count = 0,
- .dependencies = 0,
- .dependency_count = 0,
- .constructor = NULL,
- .destructor = NULL,
- .interfaces = NULL
- },
- { /* Library calls */
+static struct lcr_iface corosync_quorum_ver0[1] = {
+ {
.name = "corosync_quorum",
.version = 0,
.versions_replace = 0,
@@ -215,7 +179,6 @@
.destructor = NULL,
.interfaces = NULL,
},
-
};
static struct corosync_service_engine *quorum_get_service_handler_ver0 (void)
@@ -223,28 +186,124 @@
return (&quorum_service_handler);
}
-static struct lcr_comp vsf_quorum_comp_ver0 = {
- .iface_count = 3,
- .ifaces = corosync_vsf_quorum_ver0
+static struct lcr_comp quorum_comp_ver0 = {
+ .iface_count = 1,
+ .ifaces = corosync_quorum_ver0
};
static struct corosync_service_engine_iface_ver0 quorum_service_handler_iface = {
.corosync_get_service_engine_ver0 = quorum_get_service_handler_ver0
};
-__attribute__ ((constructor)) static void vsf_quorum_comp_register (void) {
- lcr_component_register (&vsf_quorum_comp_ver0);
- lcr_interfaces_set (&corosync_vsf_quorum_ver0[0], &vsf_quorum_iface_ver0);
- lcr_interfaces_set (&corosync_vsf_quorum_ver0[1], &quorum_service_api_v1);
- lcr_interfaces_set (&corosync_vsf_quorum_ver0[2], &quorum_service_handler_iface);
+__attribute__ ((constructor)) static void quorum_comp_register (void) {
+ lcr_component_register (&quorum_comp_ver0);
+ lcr_interfaces_set (&corosync_quorum_ver0[0], &quorum_service_handler_iface);
}
/* -------------------------------------------------- */
+
+/*
+ * Internal API functions for corosync
+ */
+
+static int quorum_quorate(void)
+{
+ return primary_designated;
+}
+
+
+static int quorum_register_callback(quorum_callback_fn_t function, void *context)
+{
+ struct internal_callback_pd *pd = malloc(sizeof(struct internal_callback_pd));
+ if (!pd)
+ return -1;
+
+ pd->context = context;
+ pd->callback = function;
+ list_add (&pd->list, &internal_trackers_list);
+
+ return 0;
+}
+
+static int quorum_unregister_callback(quorum_callback_fn_t function, void *context)
+{
+ struct internal_callback_pd *pd;
+ struct list_head *tmp;
+
+ for (tmp = internal_trackers_list.next; tmp != &internal_trackers_list; tmp = tmp->next) {
+
+ pd = list_entry(tmp, struct internal_callback_pd, list);
+ if (pd->callback == function && pd->context == context) {
+ list_del(&pd->list);
+ return 0;
+ }
+ }
+ return -1;
+}
+
+static struct quorum_callin_functions callins = {
+ .quorate = quorum_quorate,
+ .register_callback = quorum_register_callback,
+ .unregister_callback = quorum_unregister_callback
+};
+
+/* --------------------------------------------------------------------- */
+
static int quorum_exec_init_fn (struct corosync_api_v1 *api)
{
+ unsigned int find_handle;
+ unsigned int quorum_handle = 0;
+ unsigned int q_handle;
+ char *quorum_module;
+ int res;
+ void *quorum_iface_p;
+
corosync_api = api;
- list_init (&trackers_list);
+ list_init (&lib_trackers_list);
+ list_init (&internal_trackers_list);
+
+ /*
+ * Tell corosync we have a quorum engine.
+ */
+ api->quorum_initialize(&callins, &sync_primary_callback_fn);
+
+ /*
+ * Look for a quorum provider
+ */
+ api->object_find_create(OBJECT_PARENT_HANDLE, "quorum", strlen("quorum"), &find_handle);
+ api->object_find_next(find_handle, &quorum_handle);
+ api->object_find_destroy(find_handle);
+
+ if (quorum_handle) {
+ if ( !(res = api->object_key_get(quorum_handle,
+ "provider",
+ strlen("provider"),
+ (void *)&quorum_module,
+ NULL))) {
+
+ res = lcr_ifact_reference (
+ &q_handle,
+ quorum_module,
+ 0,
+ &quorum_iface_p,
+ 0);
+
+ if (res == -1) {
+ log_printf (LOG_LEVEL_NOTICE,
+ "Couldn't load quorum provider %s\n",
+ quorum_module);
+ return (-1);
+ }
+
+ log_printf (LOG_LEVEL_NOTICE,
+ "Using quorum provider %s\n", quorum_module);
+
+ quorum_iface = (struct quorum_services_api_ver1 *)quorum_iface_p;
+ quorum_iface->init (api, quorum_api_set_quorum);
+ }
+ }
+
return (0);
}
@@ -273,8 +332,22 @@
return (0);
}
-static int send_quorum_notification(void *conn)
+
+static void send_internal_notification(void)
{
+ struct list_head *tmp;
+ struct internal_callback_pd *pd;
+
+ for (tmp = internal_trackers_list.next; tmp != &internal_trackers_list; tmp = tmp->next) {
+
+ pd = list_entry(tmp, struct internal_callback_pd, list);
+
+ pd->callback(primary_designated, pd->context);
+ }
+}
+
+static void send_library_notification(void *conn)
+{
int size = sizeof(struct res_lib_quorum_notification) + sizeof(unsigned int)*quorum_view_list_entries;
char buf[size];
struct res_lib_quorum_notification *res_lib_quorum_notification = (struct res_lib_quorum_notification *)buf;
@@ -296,12 +369,12 @@
/* Send it to all interested parties */
if (conn) {
- return corosync_api->ipc_conn_send_response(conn, res_lib_quorum_notification, size);
+ corosync_api->ipc_conn_send_response(conn, res_lib_quorum_notification, size);
}
else {
struct quorum_pd *qpd;
- for (tmp = trackers_list.next; tmp != &trackers_list; tmp = tmp->next) {
+ for (tmp = lib_trackers_list.next; tmp != &lib_trackers_list; tmp = tmp->next) {
qpd = list_entry(tmp, struct quorum_pd, list);
@@ -309,7 +382,7 @@
res_lib_quorum_notification, size);
}
}
- return (0);
+ return;
}
static void message_handler_req_lib_quorum_getquorate (void *conn, void *msg)
@@ -342,7 +415,7 @@
if (req_lib_quorum_trackstart->track_flags & CS_TRACK_CURRENT ||
req_lib_quorum_trackstart->track_flags & CS_TRACK_CHANGES) {
log_printf(LOG_LEVEL_DEBUG, "sending initial status to %p\n", conn);
- send_quorum_notification(corosync_api->ipc_conn_partner_get (conn));
+ send_library_notification(corosync_api->ipc_conn_partner_get (conn));
}
/*
@@ -354,7 +427,7 @@
quorum_pd->track_flags = req_lib_quorum_trackstart->track_flags;
quorum_pd->tracking_enabled = 1;
- list_add (&quorum_pd->list, &trackers_list);
+ list_add (&quorum_pd->list, &lib_trackers_list);
}
/* send status */
Index: exec/ipc.c
===================================================================
--- exec/ipc.c (revision 1692)
+++ exec/ipc.c (working copy)
@@ -72,6 +72,7 @@
#include <corosync/engine/config.h>
#include <corosync/engine/logsys.h>
+#include "quorum.h"
#include "poll.h"
#include "totemsrp.h"
#include "mempool.h"
@@ -897,7 +898,7 @@
&send_ok_joined_iovec, 1);
send_ok =
- (sync_primary_designated() == 1 || ais_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) && (
+ (corosync_quorum_is_quorate() == 1 || ais_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) && (
(ais_service[service]->lib_engine[header->id].flow_control == CS_LIB_FLOW_CONTROL_NOT_REQUIRED) ||
((ais_service[service]->lib_engine[header->id].flow_control == CS_LIB_FLOW_CONTROL_REQUIRED) &&
(send_ok_joined) &&
Index: exec/quorum.c
===================================================================
--- exec/quorum.c (revision 0)
+++ exec/quorum.c (revision 0)
@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2008 Red Hat, Inc.
+ *
+ * All rights reserved.
+ *
+ * Author: Christine Caulfield ([EMAIL PROTECTED])
+ *
+ * This software licensed under BSD license, the text of which follows:
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright notice,
+ * this list of conditions and the following disclaimer.
+ * - Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials provided with the distribution.
+ * - Neither the name of the MontaVista Software, Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from this
+ * software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/ioctl.h>
+#include <netinet/in.h>
+#include <sys/uio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <signal.h>
+#include <time.h>
+#include <unistd.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <corosync/corotypes.h>
+#include <corosync/swab.h>
+#include <corosync/totem/totempg.h>
+#include <corosync/totem/totem.h>
+#include <corosync/lcr/lcr_ifact.h>
+#include <corosync/engine/logsys.h>
+
+#include "quorum.h"
+#include "main.h"
+#include "sync.h"
+#include "vsf.h"
+
+LOGSYS_DECLARE_SUBSYS ("QUORUM", LOG_INFO);
+
+
+static struct quorum_callin_functions *corosync_quorum_fns = NULL;
+
+int corosync_quorum_is_quorate (void)
+{
+ if (corosync_quorum_fns) {
+ return corosync_quorum_fns->quorate();
+ }
+ else {
+ return 1;
+ }
+}
+
+int corosync_quorum_register_callback (quorum_callback_fn_t fn, void *context)
+{
+ if (corosync_quorum_fns) {
+ return corosync_quorum_fns->register_callback(fn, context);
+ }
+ else {
+ return 0;
+ }
+}
+
+int corosync_quorum_unregister_callback (quorum_callback_fn_t fn, void *context)
+{
+ if (corosync_quorum_fns) {
+ return corosync_quorum_fns->unregister_callback(fn, context);
+ }
+ else {
+ return 0;
+ }
+}
+
+int corosync_quorum_initialize (struct quorum_callin_functions *fns,
+ sync_callback_fn_t *sync_callback_fn)
+{
+ if (corosync_quorum_fns)
+ return -1;
+
+ corosync_quorum_fns = fns;
+ *sync_callback_fn = sync_primary_callback_fn;
+ return 0;
+}
+
+int quorum_none(void)
+{
+ if (corosync_quorum_fns)
+ return 0;
+ else
+ return 1;
+}
Index: exec/apidef.c
===================================================================
--- exec/apidef.c (revision 1692)
+++ exec/apidef.c (working copy)
@@ -46,6 +46,7 @@
#include "main.h"
#include "ipc.h"
#include "sync.h"
+#include "quorum.h"
#include <corosync/engine/coroapi.h>
#include "service.h"
#include <corosync/lcr/lcr_ifact.h>
@@ -101,6 +102,10 @@
.tpg_groups_mcast = (typedef_tpg_groups_mcast)totempg_groups_mcast_groups,
.tpg_groups_send_ok = (typedef_tpg_groups_send_ok)totempg_groups_send_ok_groups,
.sync_request = sync_request,
+ .quorum_is_quorate = corosync_quorum_is_quorate,
+ .quorum_register_callback = corosync_quorum_register_callback,
+ .quorum_unregister_callback = corosync_quorum_unregister_callback,
+ .quorum_initialize = corosync_quorum_initialize,
.service_link_and_init = corosync_service_link_and_init,
.service_unlink_and_exit = corosync_service_unlink_and_exit,
.plugin_interface_reference = lcr_ifact_reference,
Index: exec/Makefile
===================================================================
--- exec/Makefile (revision 1692)
+++ exec/Makefile (working copy)
@@ -64,9 +64,9 @@
# main executive objects
MAIN_SRC = main.c mempool.c util.c sync.c apidef.c service.c ipc.c flow.c \
- timer.c totemconfig.c mainconfig.c
+ quorum.c timer.c totemconfig.c mainconfig.c
MAIN_OBJS = main.o mempool.o util.o sync.o apidef.o service.o ipc.o flow.o \
- timer.o totemconfig.o mainconfig.o ../lcr/lcr_ifact.o
+ quorum.o timer.o totemconfig.o mainconfig.o ../lcr/lcr_ifact.o
ifeq (${BUILD_DYNAMIC}, 1)
#EXEC_OBJS = $(TOTEM_OBJS) $(LOGSYS_OBJS) $(MAIN_OBJS)
Index: exec/quorum.h
===================================================================
--- exec/quorum.h (revision 1692)
+++ exec/quorum.h (working copy)
@@ -15,7 +15,7 @@
* - Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
- * - Neither the name of the MontaVista Software, Inc. nor the names of its
+ * - Neither the name of the Red Hat, Inc. nor the names of its
* contributors may be used to endorse or promote products derived from this
* software without specific prior written permission.
*
@@ -35,41 +35,34 @@
#ifndef QUORUM_H_DEFINED
#define QUORUM_H_DEFINED
-struct quorum_services_api_ver1 {
- void (*quorum_api_set_quorum) (unsigned int *,int,
- int, struct memb_ring_id *);
- };
+struct memb_ring_id;
-static inline struct quorum_services_api_ver1 *
-quorum_services_api_reference (
- struct corosync_api_v1 *coroapi,
- unsigned int *handle)
-{
- static void *quorum_services_api_p;
- struct quorum_services_api_ver1 *return_api;
- unsigned int res;
+typedef void (*quorum_callback_fn_t) (int quorate, void *context);
- res = coroapi->plugin_interface_reference (
- handle,
- "quorum_services_api",
- 0,
- &quorum_services_api_p,
- 0);
- if (res == -1) {
- return (NULL);
- }
- return_api = (struct quorum_services_api_ver1 *)quorum_services_api_p;
- return (return_api);
-}
+typedef void (*sync_callback_fn_t) (
+ unsigned int *view_list,
+ int view_list_entries,
+ int primary_designated,
+ struct memb_ring_id *ring_id);
-static int inline quorum_services_api_release (
- struct corosync_api_v1 *coroapi,
- unsigned int handle)
+struct quorum_callin_functions
{
- unsigned int res;
+ int (*quorate) (void);
+ int (*register_callback) (quorum_callback_fn_t, void*);
+ int (*unregister_callback) (quorum_callback_fn_t, void*);
+};
- res = coroapi->plugin_interface_release (handle);
- return (res);
-}
+extern int corosync_quorum_is_quorate (void);
+extern int corosync_quorum_register_callback (quorum_callback_fn_t fn, void *context);
+
+extern int corosync_quorum_unregister_callback (quorum_callback_fn_t fn, void *context);
+
+extern int corosync_quorum_initialize (struct quorum_callin_functions *fns,
+ sync_callback_fn_t *sync_callback_fn);
+
+
+extern int quorum_none(void);
+
+
#endif /* QUORUM_H_DEFINED */
Index: exec/vsf_ykd.c
===================================================================
--- exec/vsf_ykd.c (revision 1692)
+++ exec/vsf_ykd.c (working copy)
@@ -56,12 +56,12 @@
#include <time.h>
#include <corosync/engine/logsys.h>
+#include <corosync/ipc_gen.h>
+#include <corosync/engine/coroapi.h>
+#include <corosync/engine/quorum.h>
#include <corosync/swab.h>
#include <corosync/lcr/lcr_comp.h>
-#include "main.h"
-#include "vsf.h"
-
LOGSYS_DECLARE_SUBSYS ("YKD", LOG_INFO);
#define YKD_PROCESSOR_COUNT_MAX 32
@@ -108,7 +108,7 @@
struct ykd_state ykd_state;
-static totempg_groups_handle ykd_group_handle;
+static cs_tpg_handle ykd_group_handle;
static struct state_received state_received_confchg[YKD_PROCESSOR_COUNT_MAX];
@@ -140,6 +140,8 @@
static void *ykd_state_send_callback_token_handle = 0;
+static struct corosync_api_v1 *api;
+
static void (*ykd_primary_callback_fn) (
unsigned int *view_list,
int view_list_entries,
@@ -168,15 +170,15 @@
iovec[1].iov_base = (char *)&ykd_state;
iovec[1].iov_len = sizeof (struct ykd_state);
- res = totempg_groups_mcast_joined (ykd_group_handle, iovec, 2,
- TOTEMPG_AGREED);
+ res = api->tpg_joined_mcast (ykd_group_handle, iovec, 2,
+ TOTEM_AGREED);
return (res);
}
static void ykd_state_send (void)
{
- totempg_callback_token_create (
+ api->totem_callback_token_create (
&ykd_state_send_callback_token_handle,
TOTEM_CALLBACK_TOKEN_SENT,
1, /* delete after callback */
@@ -195,15 +197,15 @@
iovec.iov_base = (char *)&header;
iovec.iov_len = sizeof (struct ykd_header);
- res = totempg_groups_mcast_joined (ykd_group_handle, &iovec, 1,
- TOTEMPG_AGREED);
+ res = api->tpg_joined_mcast (ykd_group_handle, &iovec, 1,
+ TOTEM_AGREED);
return (res);
}
static void ykd_attempt_send (void)
{
- totempg_callback_token_create (
+ api->totem_callback_token_create (
&ykd_attempt_send_callback_token_handle,
TOTEM_CALLBACK_TOKEN_SENT,
1, /* delete after callback */
@@ -460,7 +462,7 @@
memcpy (&ykd_ring_id, ring_id, sizeof (struct memb_ring_id));
if (first_run) {
- ykd_state.last_primary.member_list[0] = totempg_my_nodeid_get();
+ ykd_state.last_primary.member_list[0] = api->totem_nodeid_get();
ykd_state.last_primary.member_list_entries = 1;
ykd_state.last_primary.session_id = 0;
first_run = 0;
@@ -493,53 +495,41 @@
ykd_state_send ();
}
-struct totempg_group ykd_group = {
+struct corosync_tpg_group ykd_group = {
.group = "ykd",
.group_len = 3
};
-static int ykd_init (
- void (*primary_callback_fn) (
- unsigned int *view_list,
- int view_list_entries,
- int primary_designated,
- struct memb_ring_id *ring_id))
+static void ykd_init (
+ struct corosync_api_v1 *corosync_api,
+ quorum_set_quorate_fn_t set_primary)
{
- ykd_primary_callback_fn = primary_callback_fn;
+ ykd_primary_callback_fn = set_primary;
+ api = corosync_api;
- totempg_groups_initialize (
+ api->tpg_init (
&ykd_group_handle,
ykd_deliver_fn,
ykd_confchg_fn);
- totempg_groups_join (
+ api->tpg_join (
ykd_group_handle,
&ykd_group,
1);
ykd_state_init ();
-
- return (0);
}
/*
- * Returns 1 if this processor is in the primary
- */
-static int ykd_primary (void) {
- return (primary_designated);
-}
-
-/*
* lcrso object definition
*/
-static struct corosync_vsf_iface_ver0 vsf_ykd_iface_ver0 = {
+static struct quorum_services_api_ver1 vsf_ykd_iface_ver0 = {
.init = ykd_init,
- .primary = ykd_primary
};
static struct lcr_iface corosync_vsf_ykd_ver0[1] = {
{
- .name = "corosync_vsf_ykd",
+ .name = "corosync_quorum_ykd",
.version = 0,
.versions_replace = 0,
.versions_replace_count = 0,
Index: exec/sync.c
===================================================================
--- exec/sync.c (revision 1692)
+++ exec/sync.c (working copy)
@@ -55,11 +55,12 @@
#include <corosync/totem/totem.h>
#include <corosync/lcr/lcr_ifact.h>
#include <corosync/engine/logsys.h>
+#include "quorum.h"
#include "main.h"
#include "sync.h"
-#include "vsf.h"
+
LOGSYS_DECLARE_SUBSYS ("SYNC", LOG_INFO);
#define MESSAGE_REQ_SYNC_BARRIER 0
@@ -72,8 +73,6 @@
static struct memb_ring_id *sync_ring_id;
-static int vsf_none = 0;
-
static int (*sync_callbacks_retrieve) (int sync_id, struct sync_callbacks *callack);
static struct sync_callbacks sync_callbacks;
@@ -93,8 +92,6 @@
static struct barrier_data barrier_data_process[PROCESSOR_COUNT_MAX];
-static struct corosync_vsf_iface_ver0 *vsf_iface;
-
static int sync_barrier_send (struct memb_ring_id *ring_id);
static int sync_start_process (enum totem_callback_token_type type, void *data);
@@ -116,12 +113,6 @@
unsigned int *joined_list, int joined_list_entries,
struct memb_ring_id *ring_id);
-static void sync_primary_callback_fn (
- unsigned int *view_list,
- int view_list_entries,
- int primary_designated,
- struct memb_ring_id *ring_id);
-
static struct totempg_group sync_group = {
.group = "sync",
.group_len = 4
@@ -266,13 +257,10 @@
int sync_register (
int (*callbacks_retrieve) (int sync_id, struct sync_callbacks *callack),
- void (*synchronization_completed) (void),
- char *vsf_type)
+ void (*synchronization_completed) (void))
+
{
unsigned int res;
- unsigned int vsf_handle;
- void *vsf_iface_p;
- char corosync_vsf_type[1024];
res = totempg_groups_initialize (
&sync_group_handle,
@@ -292,42 +280,13 @@
log_printf (LOG_LEVEL_ERROR, "Couldn't join group.\n");
return (-1);
}
-
- if (strcmp (vsf_type, "none") == 0) {
- log_printf (LOG_LEVEL_NOTICE,
- "Not using a virtual synchrony filter.\n");
- vsf_none = 1;
- } else {
- vsf_none = 0;
- sprintf (corosync_vsf_type, "corosync_vsf_%s", vsf_type);
- res = lcr_ifact_reference (
- &vsf_handle,
- corosync_vsf_type,
- 0,
- &vsf_iface_p,
- 0);
-
- if (res == -1) {
- log_printf (LOG_LEVEL_NOTICE,
- "Couldn't load virtual synchrony filter %s\n",
- vsf_type);
- return (-1);
- }
-
- log_printf (LOG_LEVEL_NOTICE,
- "Using virtual synchrony filter %s\n", corosync_vsf_type);
-
- vsf_iface = (struct corosync_vsf_iface_ver0 *)vsf_iface_p;
- vsf_iface->init (sync_primary_callback_fn);
- }
-
sync_callbacks_retrieve = callbacks_retrieve;
sync_synchronization_completed = synchronization_completed;
return (0);
}
-static void sync_primary_callback_fn (
+void sync_primary_callback_fn (
unsigned int *view_list,
int view_list_entries,
int primary_designated,
@@ -335,13 +294,6 @@
{
int i;
- if (primary_designated) {
- log_printf (LOG_LEVEL_NOTICE, "This node is within the primary component and will provide service.\n");
- } else {
- log_printf (LOG_LEVEL_NOTICE, "This node is within the non-primary component and will NOT provide any services.\n");
- return;
- }
-
/*
* Execute configuration change for synchronization service
*/
@@ -521,7 +473,7 @@
* If no virtual synchrony filter configured, then start
* synchronization process
*/
- if (vsf_none == 1) {
+ if (quorum_none() == 1) {
sync_primary_callback_fn (
member_list,
member_list_entries,
@@ -589,15 +541,6 @@
return (sync_processing);
}
-int sync_primary_designated (void)
-{
- if (vsf_none == 1) {
- return (1);
- } else {
- return (vsf_iface->primary());
- }
-}
-
/**
* Execute synchronization upon request for the named service
* @param name
Index: exec/sync.h
===================================================================
--- exec/sync.h (revision 1692)
+++ exec/sync.h (working copy)
@@ -47,10 +47,10 @@
char *name;
};
+struct corosync_api_v1;
int sync_register (
int (*sync_callbacks_retrieve) (int sync_id, struct sync_callbacks *callbacks),
- void (*synchronization_completed) (void),
- char *vsf_type);
+ void (*synchronization_completed) (void));
int sync_in_process (void);
@@ -64,4 +64,11 @@
*/
extern int sync_request (char *name);
+extern void sync_primary_callback_fn (
+ unsigned int *view_list,
+ int view_list_entries,
+ int primary_designated,
+ struct memb_ring_id *ring_id);
+
+
#endif /* SYNC_H_DEFINED */
Index: exec/main.c
===================================================================
--- exec/main.c (revision 1692)
+++ exec/main.c (working copy)
@@ -66,6 +66,7 @@
#include <corosync/engine/config.h>
#include <corosync/engine/logsys.h>
+#include "quorum.h"
#include "totemsrp.h"
#include "mempool.h"
#include "mainconfig.h"
@@ -411,7 +412,6 @@
#endif
}
-
static void deliver_fn (
unsigned int nodeid,
struct iovec *iovec,
@@ -709,10 +709,8 @@
}
- sync_register (corosync_sync_callbacks_retrieve, corosync_sync_completed,
- totem_config.vsf_type);
+ sync_register (corosync_sync_callbacks_retrieve, corosync_sync_completed);
-
res = cs_flow_control_initialize ();
/*
Index: lib/libquorum.versions
===================================================================
--- lib/libquorum.versions (revision 1692)
+++ lib/libquorum.versions (working copy)
@@ -1,3 +1,35 @@
+# Version and symbol export for libquorum.so
+
+OPENAIS_QUORUM_1.0 {
+ global:
+ quorum_initialize;
+ quorum_finalize;
+ quorum_getinfo;
+ quorum_setexpected;
+ quorum_setvotes;
+ quorum_qdisk_register;
+ quorum_qdisk_unregister;
+ quorum_qdisk_poll;
+ quorum_qdisk_getinfo;
+ quorum_setdirty;
+ quorum_killnode;
+ quorum_leaving;
+
+ local:
+ saHandleCreate;
+ saHandleDestroy;
+ saHandleInstanceGet;
+ saHandleInstancePut;
+ saRecvRetry;
+ saSelectRetry;
+ saSendMsgReceiveReply;
+ saSendMsgRetry;
+ saSendReceiveReply;
+ saSendRetry;
+ saServiceConnect;
+ saVersionVerify;
+ clustTimeNow;
+};
# Version and symbol export for libcpg.so
COROSYNC_QUORUM_1.0 {
_______________________________________________
Openais mailing list
[email protected]
https://lists.linux-foundation.org/mailman/listinfo/openais
