This is just the beginning of the quorum tests. Signed-off-by: Angus Salkeld <[email protected]> --- cts/agents/Makefile.am | 5 +- cts/agents/common_test_agent.c | 1 - cts/agents/common_test_agent.h | 6 +- cts/agents/sam_test_agent.c | 2 +- cts/agents/votequorum_test_agent.c | 222 ++++++++++++++++++++++++++++++++++++ cts/corosync.py | 27 ++++- cts/corotests.py | 216 ++++++++++++++++++++++++++++++++++- 7 files changed, 466 insertions(+), 13 deletions(-) create mode 100644 cts/agents/votequorum_test_agent.c
diff --git a/cts/agents/Makefile.am b/cts/agents/Makefile.am index be301a3..9a1b31b 100644 --- a/cts/agents/Makefile.am +++ b/cts/agents/Makefile.am @@ -33,7 +33,7 @@ MAINTAINERCLEANFILES = Makefile.in INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include SOURCES = -TEST_AGENTS = cpg_test_agent confdb_test_agent sam_test_agent +TEST_AGENTS = cpg_test_agent confdb_test_agent sam_test_agent votequorum_test_agent if INSTALL_TESTAGENTS agentdir = $(datadir)/$(PACKAGE)/tests @@ -66,6 +66,9 @@ sam_test_agent_SOURCES = sam_test_agent.c common_test_agent.c sam_test_agent_LDADD = -lsam ../../exec/coropoll.o sam_test_agent_LDFLAGS = -L../../lib +votequorum_test_agent_SOURCES = votequorum_test_agent.c common_test_agent.c +votequorum_test_agent_LDADD = -lvotequorum -lquorum -lcoroipcc ../../exec/coropoll.o +votequorum_test_agent_LDFLAGS = -L../../lib if BUILD_DARWIN diff --git a/cts/agents/common_test_agent.c b/cts/agents/common_test_agent.c index 6626103..2f1c675 100644 --- a/cts/agents/common_test_agent.c +++ b/cts/agents/common_test_agent.c @@ -47,7 +47,6 @@ #include <poll.h> #include <unistd.h> #include <fcntl.h> -#include <corosync/hdb.h> #include <corosync/totem/coropoll.h> #include "common_test_agent.h" diff --git a/cts/agents/common_test_agent.h b/cts/agents/common_test_agent.h index b717290..ca510f1 100644 --- a/cts/agents/common_test_agent.h +++ b/cts/agents/common_test_agent.h @@ -32,14 +32,15 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ +#ifndef CORO_COMMON_TEST_AGNET_H_DEFINED +#define CORO_COMMON_TEST_AGNET_H_DEFINED +#include <corosync/hdb.h> #define OK_STR "OK" #define FAIL_STR "FAIL" #define NOT_SUPPORTED_STR "NOT_SUPPORTED" - - extern int32_t parse_debug; #define HOW_BIG_AND_BUF 4096 @@ -49,4 +50,5 @@ int test_agent_run(int server_port, ta_do_command_fn func); hdb_handle_t ta_poll_handle_get(void); +#endif /* CORO_COMMON_TEST_AGNET_H_DEFINED */ diff --git a/cts/agents/sam_test_agent.c b/cts/agents/sam_test_agent.c index 6a96ad1..43c05df 100644 --- a/cts/agents/sam_test_agent.c +++ b/cts/agents/sam_test_agent.c @@ -52,9 +52,9 @@ #include <sys/un.h> #include <sys/wait.h> #include <syslog.h> + #include <corosync/corotypes.h> #include <corosync/sam.h> -#include <corosync/hdb.h> #include "common_test_agent.h" diff --git a/cts/agents/votequorum_test_agent.c b/cts/agents/votequorum_test_agent.c new file mode 100644 index 0000000..ab9d96e --- /dev/null +++ b/cts/agents/votequorum_test_agent.c @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2010 Red Hat Inc + * + * All rights reserved. + * + * Author: Angus Salkeld <[email protected]> + * + * This software licensed under BSD license, the text of which follows: + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * - Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * - Neither the name of the MontaVista Software, Inc. nor the names of its + * contributors may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <config.h> + +#include <stdio.h> +#include <stdlib.h> +#include <errno.h> +#include <unistd.h> +#include <string.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <netdb.h> +#include <sys/un.h> +#include <syslog.h> + +#include <corosync/corotypes.h> +#include <corosync/votequorum.h> +#include <corosync/quorum.h> +#include "common_test_agent.h" + + +static void getinfo (int sock) +{ + votequorum_callbacks_t callbacks; + int ret; + struct votequorum_info info; + char response[100]; + votequorum_handle_t g_handle; + + callbacks.votequorum_notify_fn = NULL; + callbacks.votequorum_expectedvotes_notify_fn = NULL; + + ret = votequorum_initialize(&g_handle, &callbacks); + if (ret != CS_OK) { + snprintf (response, 100, "%s", FAIL_STR); + syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret); + goto send_response; + } + + ret = votequorum_getinfo(g_handle, 0, &info); + if (ret != CS_OK) { + snprintf (response, 100, "%s", FAIL_STR); + syslog (LOG_ERR, "votequorum_getinfo FAILED: %d\n", ret); + goto send_response; + } + + snprintf (response, 100, "%d:%d:%d:%d:%d", + info.node_votes, + info.node_expected_votes, + info.highest_expected, + info.total_votes, + info.quorum); + +send_response: + votequorum_finalize (g_handle); + send (sock, response, strlen (response), 0); +} + + +static void setexpected (int sock, char *arg) +{ + votequorum_callbacks_t callbacks; + int ret; + char response[100]; + votequorum_handle_t g_handle; + + callbacks.votequorum_notify_fn = NULL; + callbacks.votequorum_expectedvotes_notify_fn = NULL; + + ret = votequorum_initialize(&g_handle, &callbacks); + if (ret != CS_OK) { + snprintf (response, 100, "%s", FAIL_STR); + syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret); + goto send_response; + } + + ret = votequorum_setexpected (g_handle, atoi(arg)); + if (ret != CS_OK) { + snprintf (response, 100, "%s", FAIL_STR); + syslog (LOG_ERR, "set expected votes FAILED: %d\n", ret); + goto send_response; + } + + snprintf (response, 100, "%s", OK_STR); + +send_response: + votequorum_finalize (g_handle); + send (sock, response, strlen (response) + 1, 0); +} + +static void setvotes (int sock, char *arg) +{ + votequorum_callbacks_t callbacks; + int ret; + char response[100]; + votequorum_handle_t g_handle; + + callbacks.votequorum_notify_fn = NULL; + callbacks.votequorum_expectedvotes_notify_fn = NULL; + + ret = votequorum_initialize(&g_handle, &callbacks); + if (ret != CS_OK) { + snprintf (response, 100, "%s", FAIL_STR); + syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret); + goto send_response; + } + + ret = votequorum_setvotes (g_handle, 0, atoi(arg)); + if (ret != CS_OK) { + snprintf (response, 100, "%s", FAIL_STR); + syslog (LOG_ERR, "set votes FAILED: %d\n", ret); + goto send_response; + } + + snprintf (response, 100, "%s", OK_STR); + +send_response: + votequorum_finalize (g_handle); + send (sock, response, strlen (response), 0); +} + + +static void getquorate (int sock) +{ + int ret; + int quorate; + char response[100]; + quorum_handle_t handle; + + ret = quorum_initialize (&handle, NULL); + if (ret != CS_OK) { + snprintf (response, 100, "%s", FAIL_STR); + syslog (LOG_ERR, "quorum_initialize FAILED: %d\n", ret); + goto send_response; + } + + ret = quorum_getquorate (handle, &quorate); + if (ret != CS_OK) { + snprintf (response, 100, "%s", FAIL_STR); + syslog (LOG_ERR, "getquorate FAILED: %d\n", ret); + goto send_response; + } + + snprintf (response, 100, "%d", quorate); + +send_response: + quorum_finalize (handle); + send (sock, response, strlen (response), 0); +} + +static void do_command (int sock, char* func, char*args[], int num_args) +{ + char response[100]; + + if (parse_debug) + syslog (LOG_DEBUG,"RPC:%s() called.", func); + + if (strcmp ("votequorum_getinfo", func) == 0) { + getinfo (sock); + } else if (strcmp ("votequorum_setvotes", func) == 0) { + setvotes (sock, args[0]); + } else if (strcmp ("votequorum_setexpected", func) == 0) { + setexpected (sock, args[0]); + } else if (strcmp ("quorum_getquorate", func) == 0) { + getquorate (sock); + } else { + syslog (LOG_ERR,"%s RPC:%s not supported!", __func__, func); + snprintf (response, 100, "%s", NOT_SUPPORTED_STR); + send (sock, response, strlen (response), 0); + } +} + + +int main (int argc, char *argv[]) +{ + int ret; + + openlog (NULL, LOG_CONS|LOG_PID, LOG_DAEMON); + syslog (LOG_ERR, "votequorum_test_agent STARTING"); + + parse_debug = 1; + ret = test_agent_run (9037, do_command); + syslog (LOG_ERR, "votequorum_test_agent EXITING"); + + return ret; +} + + diff --git a/cts/corosync.py b/cts/corosync.py index 4dc884c..626f8dd 100644 --- a/cts/corosync.py +++ b/cts/corosync.py @@ -135,9 +135,10 @@ class corosync_flatiron(ClusterManager): ), "LogFileName" : Environment["LogFileName"], }) - self.cpg_agent={} - self.confdb_agent={} - self.sam_agent={} + self.cpg_agent = {} + self.confdb_agent = {} + self.sam_agent = {} + self.votequorum_agent = {} self.config = CoroConfig () self.node_to_ip = {} @@ -219,6 +220,8 @@ class corosync_flatiron(ClusterManager): self.confdb_agent[node].restart() if self.sam_agent.has_key(node): self.sam_agent[node].restart() + if self.votequorum_agent.has_key(node): + self.votequorum_agent[node].restart() return ret def StopaCM(self, node): @@ -230,6 +233,8 @@ class corosync_flatiron(ClusterManager): self.cpg_agent[node].stop() if self.sam_agent.has_key(node): self.sam_agent[node].stop() + if self.votequorum_agent.has_key(node): + self.votequorum_agent[node].stop() return ClusterManager.StopaCM(self, node) def test_node_CM(self, node): @@ -334,6 +339,8 @@ class TestAgentComponent(ScenarioComponent): self.CM.confdb_agent[node].start() self.CM.sam_agent[node] = SamTestAgent(node, CM.Env) self.CM.sam_agent[node].start() + self.CM.votequorum_agent[node] = VoteQuorumTestAgent(node, CM.Env) + self.CM.votequorum_agent[node].start() return 1 def TearDown(self, CM): @@ -343,6 +350,7 @@ class TestAgentComponent(ScenarioComponent): self.CM.cpg_agent[node].stop() self.CM.confdb_agent[node].stop() self.CM.sam_agent[node].stop() + self.CM.votequorum_agent[node].stop() ################################################################### class TestAgent(object): @@ -567,3 +575,16 @@ class SamTestAgent(TestAgent): def cpg_local_get(self): return 1 +################################################################### +class VoteQuorumTestAgent(TestAgent): + + def __init__(self, node, Env=None): + TestAgent.__init__(self, "votequorum_test_agent", node, 9037, env=Env) + self.initialized = False + self.nodeid = None + self.send_recv = True + + def cpg_local_get(self): + return 1 + + diff --git a/cts/corotests.py b/cts/corotests.py index c356d86..d0a4705 100644 --- a/cts/corotests.py +++ b/cts/corotests.py @@ -45,6 +45,7 @@ class CoroTest(CTSTest): self.start = StartTest(cm) self.stop = StopTest(cm) self.config = {} + self.need_all_up = True def setup(self, node): ret = CTSTest.setup(self, node) @@ -67,11 +68,14 @@ class CoroTest(CTSTest): # apply the config self.CM.apply_new_config() - # start any killed corosync's + # start/stop all corosyncs' for n in self.CM.Env["nodes"]: - if not self.CM.StataCM(n): + if self.need_all_up and not self.CM.StataCM(n): self.incr("started") self.start(n) + if not self.need_all_up and self.CM.StataCM(n): + self.incr("stopped") + self.stop(n) return ret @@ -192,6 +196,7 @@ class CpgCfgChgOnLowestNodeJoin(CTSTest): self.start = StartTest(cm) self.stop = StopTest(cm) self.config = {} + self.need_all_up = False self.config['compatibility'] = 'none' def lowest_ip_set(self): @@ -263,7 +268,7 @@ class CpgCfgChgOnExecCrash(CpgConfigChangeBase): self.name="CpgCfgChgOnExecCrash" def failure_action(self): - self.CM.log("sending SIGSEGV to corosync on " + self.wobbly) + self.CM.log("sending KILL to corosync on " + self.wobbly) self.CM.rsh(self.wobbly, "killall -9 corosync") self.CM.rsh(self.wobbly, "rm -f /var/run/corosync.pid") self.CM.ShouldBeStatus[self.wobbly] = "down" @@ -639,6 +644,206 @@ class SamTest4(CoroTest): return self.failure('sam test 4 failed') +class QuorumState(object): + def __init__(self, cm, node): + self.node = node + self.CM = cm + + def refresh(self): + info = self.CM.votequorum_agent[self.node].votequorum_getinfo() + assert(info != 'FAIL') + assert(info != 'NOT_SUPPORTED') + + #self.CM.log('refresh: ' + info) + params = info.split(':') + + self.node_votes = int(params[0]) + self.expected_votes = int(params[1]) + self.highest_expected = int(params[2]) + self.total_votes = int(params[3]) + self.quorum = int(params[4]) + self.quorate = self.CM.votequorum_agent[self.node].quorum_getquorate() + assert(self.quorate != 'FAIL') + assert(self.quorate != 'NOT_SUPPORTED') + #self.CM.log('quorate: ' + str(self.quorate)) + +################################################################### +class VoteQuorumBase(CoroTest): + ''' + ''' + + def setup(self, node): + ret = CoroTest.setup(self, node) + self.id_map = {} + self.listener = None + for n in self.CM.Env["nodes"]: + if self.listener is None: + self.listener = n + if self.need_all_up: + self.CM.cpg_agent[n].clean_start() + self.CM.cpg_agent[n].cpg_join(self.name) + self.id_map[n] = self.CM.cpg_agent[n].cpg_local_get() + + #self.CM.votequorum_agent[self.listener].record_events() + return ret + + def wait_for_quorum_change(self): + found = False + max_timeout = 5 * 60 + waited = 0 + + printit = 0 + self.CM.log("Waiting for quorum event on " + self.listener) + while not found: + try: + event = self.CM.votequorum_agent[self.listener].read_event() + except: + return self.failure('connection to test agent failed.') + if not event == None: + self.CM.debug("RECEIVED: " + str(event)) + if event == None: + if waited >= max_timeout: + return self.failure("timedout(" + str(waited) + " sec) == no event!") + else: + time.sleep(1) + waited = waited + 1 + printit = printit + 1 + if printit is 60: + print 'waited 60 seconds' + printit = 0 + + elif str(event.node_id) in str(self.wobbly_id) and not event.is_member: + self.CM.log("Got the config change in " + str(waited) + " seconds") + found = True + else: + self.CM.debug("No match") + self.CM.debug("wobbly nodeid:" + str(self.wobbly_id)) + self.CM.debug("event nodeid:" + str(event.node_id)) + self.CM.debug("event.is_member:" + str(event.is_member)) + + if found: + return self.success() + +# repeat below with equal and uneven votes + +################################################################### +class VoteQuorumGoDown(VoteQuorumBase): +# all up +# calc min expected votes to get Q +# bring nodes down one-by-one +# confirm cluster looses Q when V < EV +# + + def __init__(self, cm): + VoteQuorumBase.__init__(self, cm) + self.name="VoteQuorumGoDown" + self.victims = [] + self.expected = len(self.CM.Env["nodes"]) + self.config['quorum/provider'] = 'corosync_votequorum' + self.config['quorum/expected_votes'] = self.expected + #self.CM.log('set expected to %d' % (self.expected)) + + def __call__(self, node): + self.incr("calls") + + state = QuorumState(self.CM, self.listener) + for n in self.CM.Env["nodes"]: + if n is self.listener: + continue + + self.victims.append(n) + self.CM.StopaCM(n) + + nodes_alive = len(self.CM.Env["nodes"]) - len(self.victims) + state.refresh() + #self.expected = self.expected - 1 + + if state.node_votes != 1: + self.failure('unexpected number of node_votes') + + if state.expected_votes != self.expected: + self.CM.log('nev: %d != exp %d' % (state.expected_votes, self.expected)) + self.failure('unexpected number of expected_votes') + + if state.total_votes != nodes_alive: + self.failure('unexpected number of total votes') + + min = ((len(self.CM.Env["nodes"]) + 2) / 2) + if min != state.quorum: + self.failure('we should have %d (not %d) as quorum' % (min, state.quorum)) + + if nodes_alive < state.quorum: + if state.quorate == 1: + self.failure('we should NOT have quorum(%d) %d > %d' % (state.quorate, state.quorum, nodes_alive)) + else: + if state.quorate == 0: + self.failure('we should have quorum(%d) %d <= %d' % (state.quorate, state.quorum, nodes_alive)) + + return self.success() + + +# all down +# calc min expected votes to get Q +# bring nodes up one-by-one +# confirm cluster gains Q when V >= EV +# +################################################################### +class VoteQuorumGoUp(VoteQuorumBase): +# all up +# calc min expected votes to get Q +# bring nodes down one-by-one +# confirm cluster looses Q when V < EV +# + + def __init__(self, cm): + VoteQuorumBase.__init__(self, cm) + self.name="VoteQuorumGoUp" + self.need_all_up = False + self.expected = len(self.CM.Env["nodes"]) + self.config['quorum/provider'] = 'corosync_votequorum' + self.config['quorum/expected_votes'] = self.expected + #self.CM.log('set expected to %d' % (self.expected)) + + def __call__(self, node): + self.incr("calls") + + self.CM.StartaCM(self.listener) + nodes_alive = 1 + state = QuorumState(self.CM, self.listener) + state.refresh() + + for n in self.CM.Env["nodes"]: + if n is self.listener: + continue + + if state.node_votes != 1: + self.failure('unexpected number of node_votes') + + if state.expected_votes != self.expected: + self.CM.log('nev: %d != exp %d' % (state.expected_votes, self.expected)) + self.failure('unexpected number of expected_votes') + + if state.total_votes != nodes_alive: + self.failure('unexpected number of total votes') + + min = ((len(self.CM.Env["nodes"]) + 2) / 2) + if min != state.quorum: + self.failure('we should have %d (not %d) as quorum' % (min, state.quorum)) + + if nodes_alive < state.quorum: + if state.quorate == 1: + self.failure('we should NOT have quorum(%d) %d > %d' % (state.quorate, state.quorum, nodes_alive)) + else: + if state.quorate == 0: + self.failure('we should have quorum(%d) %d <= %d' % (state.quorate, state.quorum, nodes_alive)) + + self.CM.StartaCM(n) + nodes_alive = nodes_alive + 1 + state.refresh() + + return self.success() + + GenTestClasses = [] GenTestClasses.append(CpgMsgOrderBasic) GenTestClasses.append(CpgMsgOrderZcb) @@ -647,6 +852,8 @@ GenTestClasses.append(CpgCfgChgOnGroupLeave) GenTestClasses.append(CpgCfgChgOnNodeLeave) GenTestClasses.append(CpgCfgChgOnNodeIsolate) GenTestClasses.append(CpgCfgChgOnLowestNodeJoin) +GenTestClasses.append(VoteQuorumGoDown) +GenTestClasses.append(VoteQuorumGoUp) AllTestClasses = [] AllTestClasses.append(ConfdbReplaceTest) @@ -657,10 +864,10 @@ AllTestClasses.append(SamTest1) AllTestClasses.append(SamTest2) AllTestClasses.append(SamTest3) AllTestClasses.append(SamTest4) - AllTestClasses.append(ServiceLoadTest) AllTestClasses.append(MemLeakObject) AllTestClasses.append(MemLeakSession) + AllTestClasses.append(FlipTest) AllTestClasses.append(RestartTest) AllTestClasses.append(StartOnebyOne) @@ -668,7 +875,6 @@ AllTestClasses.append(SimulStart) AllTestClasses.append(StopOnebyOne) AllTestClasses.append(SimulStop) AllTestClasses.append(RestartOnebyOne) -#AllTestClasses.append(PartialStart) def CoroTestList(cm, audits): -- 1.6.6.1 _______________________________________________ Openais mailing list [email protected] https://lists.linux-foundation.org/mailman/listinfo/openais
