Vladsz83 commented on a change in pull request #8159:
URL: https://github.com/apache/ignite/pull/8159#discussion_r475749342
##########
File path: modules/ducktests/tests/ignitetest/tests/discovery_test.py
##########
@@ -63,143 +75,86 @@ def __init__(self, test_context):
super(DiscoveryTest, self).__init__(test_context=test_context)
self.zk_quorum = None
self.servers = None
+ self.loader = None
- def __start_zk_quorum(self):
- self.zk_quorum = ZookeeperService(self.test_context, 3)
-
- self.stage("Starting ZooKeeper quorum")
-
- self.zk_quorum.start()
+ @cluster(num_nodes=NUM_NODES)
+ @matrix(ignite_version=[str(DEV_BRANCH), str(LATEST_2_8)],
+ kill_coordinator=[False, True],
+ nodes_to_kill=[1, 2],
+ with_load=[False, True])
+ def test_tcp(self, ignite_version, kill_coordinator, nodes_to_kill,
with_load):
+ """
+ Test nodes failure scenario with TcpDiscoverySpi.
+ """
+ config = DiscoveryTest.Config(nodes_to_kill, kill_coordinator,
with_load)
- self.stage("ZooKeeper quorum started")
+ return self.__simulate_nodes_failure(ignite_version,
self.__properties(), None, config)
- @staticmethod
- def __properties(zookeeper_settings=None):
+ @cluster(num_nodes=NUM_NODES + 3)
+ @matrix(ignite_version=[str(DEV_BRANCH), str(LATEST_2_8)],
+ kill_coordinator=[False, True],
+ nodes_to_kill=[1, 2],
+ with_load=[False, True])
+ def test_zk(self, ignite_version, kill_coordinator, nodes_to_kill,
with_load):
"""
- :param zookeeper_settings: ZooKeeperDiscoverySpi settings. If None,
TcpDiscoverySpi will be used.
- :return: Rendered node's properties.
+ Test node failure scenario with ZooKeeperSpi.
"""
- return Template(DiscoveryTest.CONFIG_TEMPLATE) \
-
.render(failure_detection_timeout=DiscoveryTest.FAILURE_DETECTION_TIMEOUT,
- zookeeper_settings=zookeeper_settings)
+ config = DiscoveryTest.Config(nodes_to_kill, kill_coordinator,
with_load)
- @staticmethod
- def __zk_properties(connection_string):
- return
DiscoveryTest.__properties(zookeeper_settings={'connection_string':
connection_string})
+ self.__start_zk_quorum()
+
+ properties = self.__zk_properties(self.zk_quorum.connection_string())
+ modules = ["zookeeper"]
+
+ return self.__simulate_nodes_failure(ignite_version, properties,
modules, config)
def setUp(self):
pass
def teardown(self):
- if self.zk_quorum:
- self.zk_quorum.stop()
+ if self.loader:
+ self.loader.stop()
if self.servers:
self.servers.stop()
- @cluster(num_nodes=NUM_NODES)
- @parametrize(version=str(DEV_BRANCH))
- @parametrize(version=str(LATEST_2_7))
- def test_tcp_not_coordinator_single(self, version):
- """
- Test single-node-failure scenario (not the coordinator) with
TcpDiscoverySpi.
- """
- return self.__simulate_nodes_failure(version, self.__properties(), 1)
-
- @cluster(num_nodes=NUM_NODES)
- @parametrize(version=str(DEV_BRANCH))
- @parametrize(version=str(LATEST_2_7))
- def test_tcp_not_coordinator_two(self, version):
- """
- Test two-node-failure scenario (not the coordinator) with
TcpDiscoverySpi.
- """
- return self.__simulate_nodes_failure(version, self.__properties(), 2)
-
- @cluster(num_nodes=NUM_NODES)
- @parametrize(version=str(DEV_BRANCH))
- @parametrize(version=str(LATEST_2_7))
- def test_tcp_coordinator(self, version):
- """
- Test coordinator-failure scenario with TcpDiscoverySpi.
- """
- return self.__simulate_nodes_failure(version, self.__properties(), 0)
-
- @cluster(num_nodes=NUM_NODES + 3)
- @parametrize(version=str(DEV_BRANCH))
- @parametrize(version=str(LATEST_2_7))
- def test_zk_not_coordinator_single(self, version):
- """
- Test single node failure scenario (not the coordinator) with ZooKeeper.
- """
- self.__start_zk_quorum()
-
- return self.__simulate_nodes_failure(version,
self.__zk_properties(self.zk_quorum.connection_string()), 1)
-
- @cluster(num_nodes=NUM_NODES + 3)
- @parametrize(version=str(DEV_BRANCH))
- @parametrize(version=str(LATEST_2_7))
- def test_zk_not_coordinator_two(self, version):
- """
- Test two-node-failure scenario (not the coordinator) with ZooKeeper.
- """
- self.__start_zk_quorum()
-
- return self.__simulate_nodes_failure(version,
self.__zk_properties(self.zk_quorum.connection_string()), 2)
-
- @cluster(num_nodes=NUM_NODES+3)
- @parametrize(version=str(DEV_BRANCH))
- @parametrize(version=str(LATEST_2_7))
- def test_zk_coordinator(self, version):
- """
- Test coordinator-failure scenario with ZooKeeper.
- """
- self.__start_zk_quorum()
+ if self.zk_quorum:
+ self.zk_quorum.stop()
- return self.__simulate_nodes_failure(version,
self.__zk_properties(self.zk_quorum.connection_string()), 0)
+ def __simulate_nodes_failure(self, version, properties, modules, config):
+ if config.nodes_to_kill == 0:
+ return {"No nodes to kill": "Nothing to do"}
- def __simulate_nodes_failure(self, version, properties, nodes_to_kill=1):
- """
- :param nodes_to_kill: How many nodes to kill. If <1, the coordinator
is the choice. Otherwise: not-coordinator
- nodes of given number.
- """
self.servers = IgniteService(
self.test_context,
- num_nodes=self.NUM_NODES,
- modules=["zookeeper"],
+ num_nodes=self.NUM_NODES - 1,
+ modules=modules,
properties=properties,
version=version)
- self.stage("Starting ignite cluster")
-
time_holder = self.monotonic()
self.servers.start()
- if nodes_to_kill > self.servers.num_nodes - 1:
- raise Exception("Too many nodes to kill: " + str(nodes_to_kill))
-
data = {'Ignite cluster start time (s)': round(self.monotonic() -
time_holder, 1)}
- self.stage("Topology is ready")
- failed_nodes, survived_node = self.__choose_node_to_kill(nodes_to_kill)
+ failed_nodes, survived_node =
self.__choose_node_to_kill(config.kill_coordinator, config.nodes_to_kill)
ids_to_wait = [node.discovery_info().node_id for node in failed_nodes]
- self.stage("Stopping " + str(len(failed_nodes)) + " nodes.")
+ if config.with_load:
+ self.__start_loading(version, properties, modules)
first_terminated = self.servers.stop_nodes_async(failed_nodes,
clean_shutdown=False, wait_for_stop=False)
- self.stage("Waiting for failure detection of " +
str(len(failed_nodes)) + " nodes.")
-
# Keeps dates of logged node failures.
logged_timestamps = []
for failed_id in ids_to_wait:
- self.servers.await_event_on_node(self.__failed_pattern(failed_id),
survived_node, 10,
+ self.servers.await_event_on_node(self.__failed_pattern(failed_id),
survived_node, 20,
from_the_beginning=True,
backoff_sec=0.01)
# Save mono of last detected failure.
time_holder = self.monotonic()
Review comment:
`first_terminated` is out the loop, it's before it. There is an atomic
inside. It holds time of the first kill command.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
