good for commit On Tue, 2010-03-30 at 13:32 +1100, Angus Salkeld wrote: > This is just the beginning of the quorum tests. > > Signed-off-by: Angus Salkeld <[email protected]> > --- > cts/agents/Makefile.am | 5 +- > cts/agents/common_test_agent.c | 1 - > cts/agents/common_test_agent.h | 6 +- > cts/agents/sam_test_agent.c | 2 +- > cts/agents/votequorum_test_agent.c | 222 > ++++++++++++++++++++++++++++++++++++ > cts/corosync.py | 27 ++++- > cts/corotests.py | 216 ++++++++++++++++++++++++++++++++++- > 7 files changed, 466 insertions(+), 13 deletions(-) > create mode 100644 cts/agents/votequorum_test_agent.c > > diff --git a/cts/agents/Makefile.am b/cts/agents/Makefile.am > index be301a3..9a1b31b 100644 > --- a/cts/agents/Makefile.am > +++ b/cts/agents/Makefile.am > @@ -33,7 +33,7 @@ MAINTAINERCLEANFILES = Makefile.in > INCLUDES = -I$(top_builddir)/include -I$(top_srcdir)/include > > SOURCES = > -TEST_AGENTS = cpg_test_agent confdb_test_agent sam_test_agent > +TEST_AGENTS = cpg_test_agent confdb_test_agent sam_test_agent > votequorum_test_agent > > if INSTALL_TESTAGENTS > agentdir = $(datadir)/$(PACKAGE)/tests > @@ -66,6 +66,9 @@ sam_test_agent_SOURCES = sam_test_agent.c > common_test_agent.c > sam_test_agent_LDADD = -lsam ../../exec/coropoll.o > sam_test_agent_LDFLAGS = -L../../lib > > +votequorum_test_agent_SOURCES = votequorum_test_agent.c common_test_agent.c > +votequorum_test_agent_LDADD = -lvotequorum -lquorum -lcoroipcc > ../../exec/coropoll.o > +votequorum_test_agent_LDFLAGS = -L../../lib > > if BUILD_DARWIN > > diff --git a/cts/agents/common_test_agent.c b/cts/agents/common_test_agent.c > index 6626103..2f1c675 100644 > --- a/cts/agents/common_test_agent.c > +++ b/cts/agents/common_test_agent.c > @@ -47,7 +47,6 @@ > #include <poll.h> > #include <unistd.h> > #include <fcntl.h> > -#include <corosync/hdb.h> > #include <corosync/totem/coropoll.h> > #include "common_test_agent.h" > > diff --git a/cts/agents/common_test_agent.h b/cts/agents/common_test_agent.h > index b717290..ca510f1 100644 > --- a/cts/agents/common_test_agent.h > +++ b/cts/agents/common_test_agent.h > @@ -32,14 +32,15 @@ > * THE POSSIBILITY OF SUCH DAMAGE. > */ > > +#ifndef CORO_COMMON_TEST_AGNET_H_DEFINED > +#define CORO_COMMON_TEST_AGNET_H_DEFINED > > +#include <corosync/hdb.h> > > #define OK_STR "OK" > #define FAIL_STR "FAIL" > #define NOT_SUPPORTED_STR "NOT_SUPPORTED" > > - > - > extern int32_t parse_debug; > #define HOW_BIG_AND_BUF 4096 > > @@ -49,4 +50,5 @@ int test_agent_run(int server_port, ta_do_command_fn func); > > hdb_handle_t ta_poll_handle_get(void); > > +#endif /* CORO_COMMON_TEST_AGNET_H_DEFINED */ > > diff --git a/cts/agents/sam_test_agent.c b/cts/agents/sam_test_agent.c > index 6a96ad1..43c05df 100644 > --- a/cts/agents/sam_test_agent.c > +++ b/cts/agents/sam_test_agent.c > @@ -52,9 +52,9 @@ > #include <sys/un.h> > #include <sys/wait.h> > #include <syslog.h> > + > #include <corosync/corotypes.h> > #include <corosync/sam.h> > -#include <corosync/hdb.h> > #include "common_test_agent.h" > > > diff --git a/cts/agents/votequorum_test_agent.c > b/cts/agents/votequorum_test_agent.c > new file mode 100644 > index 0000000..ab9d96e > --- /dev/null > +++ b/cts/agents/votequorum_test_agent.c > @@ -0,0 +1,222 @@ > +/* > + * Copyright (c) 2010 Red Hat Inc > + * > + * All rights reserved. > + * > + * Author: Angus Salkeld <[email protected]> > + * > + * This software licensed under BSD license, the text of which follows: > + * > + * Redistribution and use in source and binary forms, with or without > + * modification, are permitted provided that the following conditions are > met: > + * > + * - Redistributions of source code must retain the above copyright notice, > + * this list of conditions and the following disclaimer. > + * - Redistributions in binary form must reproduce the above copyright > notice, > + * this list of conditions and the following disclaimer in the > documentation > + * and/or other materials provided with the distribution. > + * - Neither the name of the MontaVista Software, Inc. nor the names of its > + * contributors may be used to endorse or promote products derived from > this > + * software without specific prior written permission. > + * > + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS > IS" > + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE > + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE > + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE > + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR > + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF > + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS > + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN > + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) > + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF > + * THE POSSIBILITY OF SUCH DAMAGE. > + */ > + > +#include <config.h> > + > +#include <stdio.h> > +#include <stdlib.h> > +#include <errno.h> > +#include <unistd.h> > +#include <string.h> > +#include <sys/types.h> > +#include <sys/socket.h> > +#include <netinet/in.h> > +#include <arpa/inet.h> > +#include <netdb.h> > +#include <sys/un.h> > +#include <syslog.h> > + > +#include <corosync/corotypes.h> > +#include <corosync/votequorum.h> > +#include <corosync/quorum.h> > +#include "common_test_agent.h" > + > + > +static void getinfo (int sock) > +{ > + votequorum_callbacks_t callbacks; > + int ret; > + struct votequorum_info info; > + char response[100]; > + votequorum_handle_t g_handle; > + > + callbacks.votequorum_notify_fn = NULL; > + callbacks.votequorum_expectedvotes_notify_fn = NULL; > + > + ret = votequorum_initialize(&g_handle, &callbacks); > + if (ret != CS_OK) { > + snprintf (response, 100, "%s", FAIL_STR); > + syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret); > + goto send_response; > + } > + > + ret = votequorum_getinfo(g_handle, 0, &info); > + if (ret != CS_OK) { > + snprintf (response, 100, "%s", FAIL_STR); > + syslog (LOG_ERR, "votequorum_getinfo FAILED: %d\n", ret); > + goto send_response; > + } > + > + snprintf (response, 100, "%d:%d:%d:%d:%d", > + info.node_votes, > + info.node_expected_votes, > + info.highest_expected, > + info.total_votes, > + info.quorum); > + > +send_response: > + votequorum_finalize (g_handle); > + send (sock, response, strlen (response), 0); > +} > + > + > +static void setexpected (int sock, char *arg) > +{ > + votequorum_callbacks_t callbacks; > + int ret; > + char response[100]; > + votequorum_handle_t g_handle; > + > + callbacks.votequorum_notify_fn = NULL; > + callbacks.votequorum_expectedvotes_notify_fn = NULL; > + > + ret = votequorum_initialize(&g_handle, &callbacks); > + if (ret != CS_OK) { > + snprintf (response, 100, "%s", FAIL_STR); > + syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret); > + goto send_response; > + } > + > + ret = votequorum_setexpected (g_handle, atoi(arg)); > + if (ret != CS_OK) { > + snprintf (response, 100, "%s", FAIL_STR); > + syslog (LOG_ERR, "set expected votes FAILED: %d\n", ret); > + goto send_response; > + } > + > + snprintf (response, 100, "%s", OK_STR); > + > +send_response: > + votequorum_finalize (g_handle); > + send (sock, response, strlen (response) + 1, 0); > +} > + > +static void setvotes (int sock, char *arg) > +{ > + votequorum_callbacks_t callbacks; > + int ret; > + char response[100]; > + votequorum_handle_t g_handle; > + > + callbacks.votequorum_notify_fn = NULL; > + callbacks.votequorum_expectedvotes_notify_fn = NULL; > + > + ret = votequorum_initialize(&g_handle, &callbacks); > + if (ret != CS_OK) { > + snprintf (response, 100, "%s", FAIL_STR); > + syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret); > + goto send_response; > + } > + > + ret = votequorum_setvotes (g_handle, 0, atoi(arg)); > + if (ret != CS_OK) { > + snprintf (response, 100, "%s", FAIL_STR); > + syslog (LOG_ERR, "set votes FAILED: %d\n", ret); > + goto send_response; > + } > + > + snprintf (response, 100, "%s", OK_STR); > + > +send_response: > + votequorum_finalize (g_handle); > + send (sock, response, strlen (response), 0); > +} > + > + > +static void getquorate (int sock) > +{ > + int ret; > + int quorate; > + char response[100]; > + quorum_handle_t handle; > + > + ret = quorum_initialize (&handle, NULL); > + if (ret != CS_OK) { > + snprintf (response, 100, "%s", FAIL_STR); > + syslog (LOG_ERR, "quorum_initialize FAILED: %d\n", ret); > + goto send_response; > + } > + > + ret = quorum_getquorate (handle, &quorate); > + if (ret != CS_OK) { > + snprintf (response, 100, "%s", FAIL_STR); > + syslog (LOG_ERR, "getquorate FAILED: %d\n", ret); > + goto send_response; > + } > + > + snprintf (response, 100, "%d", quorate); > + > +send_response: > + quorum_finalize (handle); > + send (sock, response, strlen (response), 0); > +} > + > +static void do_command (int sock, char* func, char*args[], int num_args) > +{ > + char response[100]; > + > + if (parse_debug) > + syslog (LOG_DEBUG,"RPC:%s() called.", func); > + > + if (strcmp ("votequorum_getinfo", func) == 0) { > + getinfo (sock); > + } else if (strcmp ("votequorum_setvotes", func) == 0) { > + setvotes (sock, args[0]); > + } else if (strcmp ("votequorum_setexpected", func) == 0) { > + setexpected (sock, args[0]); > + } else if (strcmp ("quorum_getquorate", func) == 0) { > + getquorate (sock); > + } else { > + syslog (LOG_ERR,"%s RPC:%s not supported!", __func__, func); > + snprintf (response, 100, "%s", NOT_SUPPORTED_STR); > + send (sock, response, strlen (response), 0); > + } > +} > + > + > +int main (int argc, char *argv[]) > +{ > + int ret; > + > + openlog (NULL, LOG_CONS|LOG_PID, LOG_DAEMON); > + syslog (LOG_ERR, "votequorum_test_agent STARTING"); > + > + parse_debug = 1; > + ret = test_agent_run (9037, do_command); > + syslog (LOG_ERR, "votequorum_test_agent EXITING"); > + > + return ret; > +} > + > + > diff --git a/cts/corosync.py b/cts/corosync.py > index 4dc884c..626f8dd 100644 > --- a/cts/corosync.py > +++ b/cts/corosync.py > @@ -135,9 +135,10 @@ class corosync_flatiron(ClusterManager): > ), > "LogFileName" : Environment["LogFileName"], > }) > - self.cpg_agent={} > - self.confdb_agent={} > - self.sam_agent={} > + self.cpg_agent = {} > + self.confdb_agent = {} > + self.sam_agent = {} > + self.votequorum_agent = {} > self.config = CoroConfig () > self.node_to_ip = {} > > @@ -219,6 +220,8 @@ class corosync_flatiron(ClusterManager): > self.confdb_agent[node].restart() > if self.sam_agent.has_key(node): > self.sam_agent[node].restart() > + if self.votequorum_agent.has_key(node): > + self.votequorum_agent[node].restart() > return ret > > def StopaCM(self, node): > @@ -230,6 +233,8 @@ class corosync_flatiron(ClusterManager): > self.cpg_agent[node].stop() > if self.sam_agent.has_key(node): > self.sam_agent[node].stop() > + if self.votequorum_agent.has_key(node): > + self.votequorum_agent[node].stop() > return ClusterManager.StopaCM(self, node) > > def test_node_CM(self, node): > @@ -334,6 +339,8 @@ class TestAgentComponent(ScenarioComponent): > self.CM.confdb_agent[node].start() > self.CM.sam_agent[node] = SamTestAgent(node, CM.Env) > self.CM.sam_agent[node].start() > + self.CM.votequorum_agent[node] = VoteQuorumTestAgent(node, > CM.Env) > + self.CM.votequorum_agent[node].start() > return 1 > > def TearDown(self, CM): > @@ -343,6 +350,7 @@ class TestAgentComponent(ScenarioComponent): > self.CM.cpg_agent[node].stop() > self.CM.confdb_agent[node].stop() > self.CM.sam_agent[node].stop() > + self.CM.votequorum_agent[node].stop() > > ################################################################### > class TestAgent(object): > @@ -567,3 +575,16 @@ class SamTestAgent(TestAgent): > def cpg_local_get(self): > return 1 > > +################################################################### > +class VoteQuorumTestAgent(TestAgent): > + > + def __init__(self, node, Env=None): > + TestAgent.__init__(self, "votequorum_test_agent", node, 9037, > env=Env) > + self.initialized = False > + self.nodeid = None > + self.send_recv = True > + > + def cpg_local_get(self): > + return 1 > + > + > diff --git a/cts/corotests.py b/cts/corotests.py > index c356d86..d0a4705 100644 > --- a/cts/corotests.py > +++ b/cts/corotests.py > @@ -45,6 +45,7 @@ class CoroTest(CTSTest): > self.start = StartTest(cm) > self.stop = StopTest(cm) > self.config = {} > + self.need_all_up = True > > def setup(self, node): > ret = CTSTest.setup(self, node) > @@ -67,11 +68,14 @@ class CoroTest(CTSTest): > # apply the config > self.CM.apply_new_config() > > - # start any killed corosync's > + # start/stop all corosyncs' > for n in self.CM.Env["nodes"]: > - if not self.CM.StataCM(n): > + if self.need_all_up and not self.CM.StataCM(n): > self.incr("started") > self.start(n) > + if not self.need_all_up and self.CM.StataCM(n): > + self.incr("stopped") > + self.stop(n) > return ret > > > @@ -192,6 +196,7 @@ class CpgCfgChgOnLowestNodeJoin(CTSTest): > self.start = StartTest(cm) > self.stop = StopTest(cm) > self.config = {} > + self.need_all_up = False > self.config['compatibility'] = 'none' > > def lowest_ip_set(self): > @@ -263,7 +268,7 @@ class CpgCfgChgOnExecCrash(CpgConfigChangeBase): > self.name="CpgCfgChgOnExecCrash" > > def failure_action(self): > - self.CM.log("sending SIGSEGV to corosync on " + self.wobbly) > + self.CM.log("sending KILL to corosync on " + self.wobbly) > self.CM.rsh(self.wobbly, "killall -9 corosync") > self.CM.rsh(self.wobbly, "rm -f /var/run/corosync.pid") > self.CM.ShouldBeStatus[self.wobbly] = "down" > @@ -639,6 +644,206 @@ class SamTest4(CoroTest): > return self.failure('sam test 4 failed') > > > +class QuorumState(object): > + def __init__(self, cm, node): > + self.node = node > + self.CM = cm > + > + def refresh(self): > + info = self.CM.votequorum_agent[self.node].votequorum_getinfo() > + assert(info != 'FAIL') > + assert(info != 'NOT_SUPPORTED') > + > + #self.CM.log('refresh: ' + info) > + params = info.split(':') > + > + self.node_votes = int(params[0]) > + self.expected_votes = int(params[1]) > + self.highest_expected = int(params[2]) > + self.total_votes = int(params[3]) > + self.quorum = int(params[4]) > + self.quorate = > self.CM.votequorum_agent[self.node].quorum_getquorate() > + assert(self.quorate != 'FAIL') > + assert(self.quorate != 'NOT_SUPPORTED') > + #self.CM.log('quorate: ' + str(self.quorate)) > + > +################################################################### > +class VoteQuorumBase(CoroTest): > + ''' > + ''' > + > + def setup(self, node): > + ret = CoroTest.setup(self, node) > + self.id_map = {} > + self.listener = None > + for n in self.CM.Env["nodes"]: > + if self.listener is None: > + self.listener = n > + if self.need_all_up: > + self.CM.cpg_agent[n].clean_start() > + self.CM.cpg_agent[n].cpg_join(self.name) > + self.id_map[n] = self.CM.cpg_agent[n].cpg_local_get() > + > + #self.CM.votequorum_agent[self.listener].record_events() > + return ret > + > + def wait_for_quorum_change(self): > + found = False > + max_timeout = 5 * 60 > + waited = 0 > + > + printit = 0 > + self.CM.log("Waiting for quorum event on " + self.listener) > + while not found: > + try: > + event = self.CM.votequorum_agent[self.listener].read_event() > + except: > + return self.failure('connection to test agent failed.') > + if not event == None: > + self.CM.debug("RECEIVED: " + str(event)) > + if event == None: > + if waited >= max_timeout: > + return self.failure("timedout(" + str(waited) + " sec) > == no event!") > + else: > + time.sleep(1) > + waited = waited + 1 > + printit = printit + 1 > + if printit is 60: > + print 'waited 60 seconds' > + printit = 0 > + > + elif str(event.node_id) in str(self.wobbly_id) and not > event.is_member: > + self.CM.log("Got the config change in " + str(waited) + " > seconds") > + found = True > + else: > + self.CM.debug("No match") > + self.CM.debug("wobbly nodeid:" + str(self.wobbly_id)) > + self.CM.debug("event nodeid:" + str(event.node_id)) > + self.CM.debug("event.is_member:" + str(event.is_member)) > + > + if found: > + return self.success() > + > +# repeat below with equal and uneven votes > + > +################################################################### > +class VoteQuorumGoDown(VoteQuorumBase): > +# all up > +# calc min expected votes to get Q > +# bring nodes down one-by-one > +# confirm cluster looses Q when V < EV > +# > + > + def __init__(self, cm): > + VoteQuorumBase.__init__(self, cm) > + self.name="VoteQuorumGoDown" > + self.victims = [] > + self.expected = len(self.CM.Env["nodes"]) > + self.config['quorum/provider'] = 'corosync_votequorum' > + self.config['quorum/expected_votes'] = self.expected > + #self.CM.log('set expected to %d' % (self.expected)) > + > + def __call__(self, node): > + self.incr("calls") > + > + state = QuorumState(self.CM, self.listener) > + for n in self.CM.Env["nodes"]: > + if n is self.listener: > + continue > + > + self.victims.append(n) > + self.CM.StopaCM(n) > + > + nodes_alive = len(self.CM.Env["nodes"]) - len(self.victims) > + state.refresh() > + #self.expected = self.expected - 1 > + > + if state.node_votes != 1: > + self.failure('unexpected number of node_votes') > + > + if state.expected_votes != self.expected: > + self.CM.log('nev: %d != exp %d' % (state.expected_votes, > self.expected)) > + self.failure('unexpected number of expected_votes') > + > + if state.total_votes != nodes_alive: > + self.failure('unexpected number of total votes') > + > + min = ((len(self.CM.Env["nodes"]) + 2) / 2) > + if min != state.quorum: > + self.failure('we should have %d (not %d) as quorum' % (min, > state.quorum)) > + > + if nodes_alive < state.quorum: > + if state.quorate == 1: > + self.failure('we should NOT have quorum(%d) %d > %d' % > (state.quorate, state.quorum, nodes_alive)) > + else: > + if state.quorate == 0: > + self.failure('we should have quorum(%d) %d <= %d' % > (state.quorate, state.quorum, nodes_alive)) > + > + return self.success() > + > + > +# all down > +# calc min expected votes to get Q > +# bring nodes up one-by-one > +# confirm cluster gains Q when V >= EV > +# > +################################################################### > +class VoteQuorumGoUp(VoteQuorumBase): > +# all up > +# calc min expected votes to get Q > +# bring nodes down one-by-one > +# confirm cluster looses Q when V < EV > +# > + > + def __init__(self, cm): > + VoteQuorumBase.__init__(self, cm) > + self.name="VoteQuorumGoUp" > + self.need_all_up = False > + self.expected = len(self.CM.Env["nodes"]) > + self.config['quorum/provider'] = 'corosync_votequorum' > + self.config['quorum/expected_votes'] = self.expected > + #self.CM.log('set expected to %d' % (self.expected)) > + > + def __call__(self, node): > + self.incr("calls") > + > + self.CM.StartaCM(self.listener) > + nodes_alive = 1 > + state = QuorumState(self.CM, self.listener) > + state.refresh() > + > + for n in self.CM.Env["nodes"]: > + if n is self.listener: > + continue > + > + if state.node_votes != 1: > + self.failure('unexpected number of node_votes') > + > + if state.expected_votes != self.expected: > + self.CM.log('nev: %d != exp %d' % (state.expected_votes, > self.expected)) > + self.failure('unexpected number of expected_votes') > + > + if state.total_votes != nodes_alive: > + self.failure('unexpected number of total votes') > + > + min = ((len(self.CM.Env["nodes"]) + 2) / 2) > + if min != state.quorum: > + self.failure('we should have %d (not %d) as quorum' % (min, > state.quorum)) > + > + if nodes_alive < state.quorum: > + if state.quorate == 1: > + self.failure('we should NOT have quorum(%d) %d > %d' % > (state.quorate, state.quorum, nodes_alive)) > + else: > + if state.quorate == 0: > + self.failure('we should have quorum(%d) %d <= %d' % > (state.quorate, state.quorum, nodes_alive)) > + > + self.CM.StartaCM(n) > + nodes_alive = nodes_alive + 1 > + state.refresh() > + > + return self.success() > + > + > GenTestClasses = [] > GenTestClasses.append(CpgMsgOrderBasic) > GenTestClasses.append(CpgMsgOrderZcb) > @@ -647,6 +852,8 @@ GenTestClasses.append(CpgCfgChgOnGroupLeave) > GenTestClasses.append(CpgCfgChgOnNodeLeave) > GenTestClasses.append(CpgCfgChgOnNodeIsolate) > GenTestClasses.append(CpgCfgChgOnLowestNodeJoin) > +GenTestClasses.append(VoteQuorumGoDown) > +GenTestClasses.append(VoteQuorumGoUp) > > AllTestClasses = [] > AllTestClasses.append(ConfdbReplaceTest) > @@ -657,10 +864,10 @@ AllTestClasses.append(SamTest1) > AllTestClasses.append(SamTest2) > AllTestClasses.append(SamTest3) > AllTestClasses.append(SamTest4) > - > AllTestClasses.append(ServiceLoadTest) > AllTestClasses.append(MemLeakObject) > AllTestClasses.append(MemLeakSession) > + > AllTestClasses.append(FlipTest) > AllTestClasses.append(RestartTest) > AllTestClasses.append(StartOnebyOne) > @@ -668,7 +875,6 @@ AllTestClasses.append(SimulStart) > AllTestClasses.append(StopOnebyOne) > AllTestClasses.append(SimulStop) > AllTestClasses.append(RestartOnebyOne) > -#AllTestClasses.append(PartialStart) > > > def CoroTestList(cm, audits):
_______________________________________________ Openais mailing list [email protected] https://lists.linux-foundation.org/mailman/listinfo/openais
