Hello community, here is the log from the commit of package crmsh for openSUSE:Factory checked in at 2020-12-04 21:29:20 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/crmsh (Old) and /work/SRC/openSUSE:Factory/.crmsh.new.5913 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "crmsh" Fri Dec 4 21:29:20 2020 rev:198 rq:853078 version:4.2.0+git.1607075079.a25648d8 Changes: -------- --- /work/SRC/openSUSE:Factory/crmsh/crmsh.changes 2020-12-03 18:43:54.326250951 +0100 +++ /work/SRC/openSUSE:Factory/.crmsh.new.5913/crmsh.changes 2020-12-04 21:29:21.318199506 +0100 @@ -1,0 +2,7 @@ +Fri Dec 04 09:56:18 UTC 2020 - [email protected] + +- Update to version 4.2.0+git.1607075079.a25648d8: + * Dev: unittest: unit test code for class bootstrap.JoinLock + * Fix: bootstrap: use class JoinLock to manage lock in parallel join(bsc#1175976) + +------------------------------------------------------------------- Old: ---- crmsh-4.2.0+git.1606837217.bf7af3a6.tar.bz2 New: ---- crmsh-4.2.0+git.1607075079.a25648d8.tar.bz2 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ crmsh.spec ++++++ --- /var/tmp/diff_new_pack.6oroZ9/_old 2020-12-04 21:29:22.242200831 +0100 +++ /var/tmp/diff_new_pack.6oroZ9/_new 2020-12-04 21:29:22.246200836 +0100 @@ -36,7 +36,7 @@ Summary: High Availability cluster command-line interface License: GPL-2.0-or-later Group: %{pkg_group} -Version: 4.2.0+git.1606837217.bf7af3a6 +Version: 4.2.0+git.1607075079.a25648d8 Release: 0 Url: http://crmsh.github.io Source0: %{name}-%{version}.tar.bz2 ++++++ _servicedata ++++++ --- /var/tmp/diff_new_pack.6oroZ9/_old 2020-12-04 21:29:22.286200895 +0100 +++ /var/tmp/diff_new_pack.6oroZ9/_new 2020-12-04 21:29:22.286200895 +0100 @@ -5,4 +5,4 @@ <param name="url">https://github.com/liangxin1300/crmsh.git</param> <param name="changesrevision">d8dc51b4cb34964aa72e918999ebc7f03b48f3c9</param></service><service name="tar_scm"> <param name="url">https://github.com/ClusterLabs/crmsh.git</param> - <param name="changesrevision">715b9035cf2bc766d3809a8fe85bbeccf63a93b9</param></service></servicedata> \ No newline at end of file + <param name="changesrevision">f70e90d25bc286997f9ca427d86390a7528b064c</param></service></servicedata> \ No newline at end of file ++++++ crmsh-4.2.0+git.1606837217.bf7af3a6.tar.bz2 -> crmsh-4.2.0+git.1607075079.a25648d8.tar.bz2 ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/crmsh-4.2.0+git.1606837217.bf7af3a6/crm.conf.in new/crmsh-4.2.0+git.1607075079.a25648d8/crm.conf.in --- old/crmsh-4.2.0+git.1606837217.bf7af3a6/crm.conf.in 2020-12-01 16:40:17.000000000 +0100 +++ new/crmsh-4.2.0+git.1607075079.a25648d8/crm.conf.in 2020-12-04 10:44:39.000000000 +0100 @@ -19,6 +19,7 @@ ; dot = dot ; ignore_missing_metadata = no ; report_tool_options = +; join_timeout = 120 ; obscure_pattern option is the persisent configuration of CLI. ; Example, for the high security concern, obscure_pattern = passw* | ip diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/crmsh-4.2.0+git.1606837217.bf7af3a6/crmsh/bootstrap.py new/crmsh-4.2.0+git.1607075079.a25648d8/crmsh/bootstrap.py --- old/crmsh-4.2.0+git.1606837217.bf7af3a6/crmsh/bootstrap.py 2020-12-01 16:40:17.000000000 +0100 +++ new/crmsh-4.2.0+git.1607075079.a25648d8/crmsh/bootstrap.py 2020-12-04 10:44:39.000000000 +0100 @@ -30,6 +30,7 @@ from . import tmpfiles from . import clidisplay from . import term +from . import join_lock LOG_FILE = "/var/log/crmsh/ha-cluster-bootstrap.log" @@ -1711,8 +1712,6 @@ if not invoke("ssh root@{} crm cluster init -i {} ssh_remote".format(seed_host, _context.default_nic_list[0])): error("Can't invoke crm cluster init -i {} ssh_remote on {}".format(_context.default_nic_list[0], seed_host)) - setup_passwordless_with_other_nodes(seed_host) - def swap_public_ssh_key(remote_node): """ @@ -2338,10 +2337,14 @@ _context.cluster_node = cluster_node join_ssh(cluster_node) - join_remote_auth(cluster_node) - join_csync2(cluster_node) - join_ssh_merge(cluster_node) - join_cluster(cluster_node) + + lock_inst = join_lock.JoinLock(cluster_node) + with lock_inst.lock(): + setup_passwordless_with_other_nodes(cluster_node) + join_remote_auth(cluster_node) + join_csync2(cluster_node) + join_ssh_merge(cluster_node) + join_cluster(cluster_node) status("Done (log saved to %s)" % (LOG_FILE)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/crmsh-4.2.0+git.1606837217.bf7af3a6/crmsh/config.py new/crmsh-4.2.0+git.1607075079.a25648d8/crmsh/config.py --- old/crmsh-4.2.0+git.1606837217.bf7af3a6/crmsh/config.py 2020-12-01 16:40:17.000000000 +0100 +++ new/crmsh-4.2.0+git.1607075079.a25648d8/crmsh/config.py 2020-12-04 10:44:39.000000000 +0100 @@ -239,6 +239,7 @@ 'dot': opt_program('', ('dot',)), 'ignore_missing_metadata': opt_boolean('no'), 'report_tool_options': opt_string(''), + 'join_timeout': opt_string('120'), 'obscure_pattern': opt_string('passw*') }, 'path': { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/crmsh-4.2.0+git.1606837217.bf7af3a6/crmsh/join_lock.py new/crmsh-4.2.0+git.1607075079.a25648d8/crmsh/join_lock.py --- old/crmsh-4.2.0+git.1606837217.bf7af3a6/crmsh/join_lock.py 1970-01-01 01:00:00.000000000 +0100 +++ new/crmsh-4.2.0+git.1607075079.a25648d8/crmsh/join_lock.py 2020-12-04 10:44:39.000000000 +0100 @@ -0,0 +1,140 @@ +# Copyright (C) 2020 Xin Liang <[email protected]> +# See COPYING for license information. + + +import re +import time +from contextlib import contextmanager + +from . import bootstrap +from . import utils +from . import config + + +class SSHError(Exception): + pass + + +class JoinLock(object): + """ + Class to manage lock for multiple nodes join in parallel + """ + + JOIN_LOCK_DIR = "/tmp/.crmsh_join_lock_directory" + MKDIR_CMD = "mkdir {}".format(JOIN_LOCK_DIR) + RM_CMD = "rm -rf {}".format(JOIN_LOCK_DIR) + SSH_TIMEOUT = 10 + SSH_OPTION = "-o ConnectTimeout={} -o StrictHostKeyChecking=no".format(SSH_TIMEOUT) + SSH_EXIT_ERR = 255 + MIN_JOIN_TIMEOUT = 120 + WAIT_INTERVAL = 10 + + def __init__(self, init_node): + """ + Init function + """ + self.init_node = init_node + # only the lock owner can unlock + self.lock_owner = False + + @property + def join_timeout(self): + """ + Get join_timeout from config.core + """ + try: + value = int(config.core.join_timeout) + except ValueError: + raise ValueError("Invalid format of core.join_timeout(should be a number)") + if value < self.MIN_JOIN_TIMEOUT: + raise ValueError("Minimum value of core.join_timeout should be {}".format(self.MIN_JOIN_TIMEOUT)) + return value + + def _run(self, cmd): + """ + Run command on target node, consider specific exceptions + """ + cmd_with_ssh = "ssh {} root@{} \"{}\"".format(self.SSH_OPTION, self.init_node, cmd) + rc, out, err = utils.get_stdout_stderr(cmd_with_ssh) + if rc == self.SSH_EXIT_ERR: + raise SSHError(err) + return rc, out, err + + def _create_lock_dir(self): + """ + Create lock directory, mkdir command was atomic + """ + rc, _, _ = self._run(self.MKDIR_CMD) + return rc == 0 + + def _get_online_nodelist(self): + """ + Get the online node list from init node + """ + rc, out, err = self._run("crm_node -l") + if rc != 0 and err: + raise RuntimeError(err) + return re.findall('[0-9]+ (.*) member', out) + + def _lock_or_wait(self): + """ + Try to claim lock on init node, + wait if failed to claim + exit if reached the join_timeout + """ + warned_once = False + online_list = [] + pre_online_list = [] + expired_error_str = "Cannot continue since the lock directory exists at the init node ({}:{})".format(self.init_node, self.JOIN_LOCK_DIR) + + current_time = int(time.time()) + timeout = current_time + self.join_timeout + while current_time <= timeout: + + # Try to claim the lock + if self._create_lock_dir(): + # Success + self.lock_owner = True + break + + # Might lose claiming lock again, start to wait again + online_list = self._get_online_nodelist() + if pre_online_list and pre_online_list != online_list: + warned_once = False + current_time = int(time.time()) + timeout = current_time + self.join_timeout + continue + else: + pre_online_list = online_list + + if not warned_once: + warned_once = True + bootstrap.warn("Other node still joining, wait at most {}s...".format(self.join_timeout)) + + time.sleep(self.WAIT_INTERVAL) + current_time = int(time.time()) + + else: + raise TimeoutError("Join process failed after {} seconds. {}".format(self.join_timeout, expired_error_str)) + + @contextmanager + def lock(self): + """ + Create lock directory on target node + """ + try: + self._lock_or_wait() + yield + except SSHError as err: + bootstrap.error(str(err)) + except: + raise + finally: + self.unlock() + + def unlock(self): + """ + Remove the lock directory on target node + """ + if self.lock_owner: + self._run(self.RM_CMD) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/crmsh-4.2.0+git.1606837217.bf7af3a6/data-manifest new/crmsh-4.2.0+git.1607075079.a25648d8/data-manifest --- old/crmsh-4.2.0+git.1606837217.bf7af3a6/data-manifest 2020-12-01 16:40:17.000000000 +0100 +++ new/crmsh-4.2.0+git.1607075079.a25648d8/data-manifest 2020-12-04 10:44:39.000000000 +0100 @@ -184,12 +184,14 @@ test/unittests/test_corosync.py test/unittests/test_gv.py test/unittests/test_handles.py +test/unittests/test_join_lock.py test/unittests/test_objset.py test/unittests/test_parallax.py test/unittests/test_parse.py test/unittests/test_report.py test/unittests/test_scripts.py test/unittests/test_time.py +test/unittests/test_ui_cluster.py test/unittests/test_utils.py test/update-expected-output.sh utils/crm_clean.py diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/crmsh-4.2.0+git.1606837217.bf7af3a6/test/unittests/test_bootstrap.py new/crmsh-4.2.0+git.1607075079.a25648d8/test/unittests/test_bootstrap.py --- old/crmsh-4.2.0+git.1606837217.bf7af3a6/test/unittests/test_bootstrap.py 2020-12-01 16:40:17.000000000 +0100 +++ new/crmsh-4.2.0+git.1607075079.a25648d8/test/unittests/test_bootstrap.py 2020-12-04 10:44:39.000000000 +0100 @@ -579,13 +579,12 @@ bootstrap.join_ssh(None) mock_error.assert_called_once_with("No existing IP/hostname specified (use -c option)") - @mock.patch('crmsh.bootstrap.setup_passwordless_with_other_nodes') @mock.patch('crmsh.bootstrap.error') @mock.patch('crmsh.bootstrap.invoke') @mock.patch('crmsh.bootstrap.swap_public_ssh_key') @mock.patch('crmsh.bootstrap.configure_local_ssh_key') @mock.patch('crmsh.utils.start_service') - def test_join_ssh(self, mock_start_service, mock_config_ssh, mock_swap, mock_invoke, mock_error, mock_swap_other): + def test_join_ssh(self, mock_start_service, mock_config_ssh, mock_swap, mock_invoke, mock_error): bootstrap._context = mock.Mock(default_nic_list=["eth1"]) mock_invoke.return_value = False @@ -596,7 +595,6 @@ mock_swap.assert_called_once_with("node1") mock_invoke.assert_called_once_with("ssh root@node1 crm cluster init -i eth1 ssh_remote") mock_error.assert_called_once_with("Can't invoke crm cluster init -i eth1 ssh_remote on node1") - mock_swap_other.assert_called_once_with("node1") @mock.patch('crmsh.bootstrap.warn') @mock.patch('crmsh.bootstrap.fetch_public_key_from_remote_node') diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/crmsh-4.2.0+git.1606837217.bf7af3a6/test/unittests/test_join_lock.py new/crmsh-4.2.0+git.1607075079.a25648d8/test/unittests/test_join_lock.py --- old/crmsh-4.2.0+git.1606837217.bf7af3a6/test/unittests/test_join_lock.py 1970-01-01 01:00:00.000000000 +0100 +++ new/crmsh-4.2.0+git.1607075079.a25648d8/test/unittests/test_join_lock.py 2020-12-04 10:44:39.000000000 +0100 @@ -0,0 +1,206 @@ +""" +Unitary tests for crmsh/join_lock.py + +:author: xinliang +:organization: SUSE Linux GmbH +:contact: [email protected] + +:since: 2020-11-15 +""" + +# pylint:disable=C0103,C0111,W0212,W0611 + +import os +import unittest + +try: + from unittest import mock +except ImportError: + import mock + +from crmsh import join_lock, config + + +class TestJoinLock(unittest.TestCase): + """ + Unitary tests for crmsh.join_lock.JoinLock + """ + + @classmethod + def setUpClass(cls): + """ + Global setUp. + """ + + def setUp(self): + """ + Test setUp. + """ + self.lock_inst = join_lock.JoinLock("node1") + + def tearDown(self): + """ + Test tearDown. + """ + + @classmethod + def tearDownClass(cls): + """ + Global tearDown. + """ + + def test_join_timeout_error_format(self): + config.core.join_timeout = "pwd" + with self.assertRaises(ValueError) as err: + self.lock_inst.join_timeout + self.assertEqual("Invalid format of core.join_timeout(should be a number)", str(err.exception)) + + def test_join_timeout_min_error(self): + config.core.join_timeout = "12" + with self.assertRaises(ValueError) as err: + self.lock_inst.join_timeout + self.assertEqual("Minimum value of core.join_timeout should be 120", str(err.exception)) + + def test_join_timeout(self): + config.core.join_timeout = "130" + self.assertEqual(self.lock_inst.join_timeout, 130) + + @mock.patch('crmsh.utils.get_stdout_stderr') + def test_run_error(self, mock_run): + mock_run.return_value = (255, "output", "error data") + with self.assertRaises(join_lock.SSHError) as err: + self.lock_inst._run("test_cmd") + self.assertEqual("error data", str(err.exception)) + mock_run.assert_called_once_with('ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@node1 "test_cmd"') + + @mock.patch('crmsh.utils.get_stdout_stderr') + def test_run(self, mock_run): + mock_run.return_value = (0, "output data", None) + rc, out, err = self.lock_inst._run("test_cmd") + self.assertEqual(mock_run.return_value, (rc, out, err)) + mock_run.assert_called_once_with('ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@node1 "test_cmd"') + + @mock.patch('crmsh.join_lock.JoinLock._run') + def test_create_lock_dir(self, mock_run): + mock_run.return_value = (0, None, None) + rc = self.lock_inst._create_lock_dir() + self.assertEqual(rc, True) + mock_run.assert_called_once_with(join_lock.JoinLock.MKDIR_CMD) + + @mock.patch('crmsh.join_lock.JoinLock._run') + def test_get_online_nodelist_error(self, mock_run): + mock_run.return_value = (1, None, "error data") + with self.assertRaises(RuntimeError) as err: + self.lock_inst._get_online_nodelist() + self.assertEqual("error data", str(err.exception)) + mock_run.assert_called_once_with("crm_node -l") + + @mock.patch('crmsh.join_lock.JoinLock._run') + def test_get_online_nodelist(self, mock_run): + output = """ + 1084783297 15sp2-1 member + 1084783193 15sp2-2 lost + 1084783331 15sp2-3 member + """ + mock_run.return_value = (0, output, None) + res = self.lock_inst._get_online_nodelist() + self.assertEqual(res, ["15sp2-1", "15sp2-3"]) + mock_run.assert_called_once_with("crm_node -l") + + @mock.patch('crmsh.join_lock.JoinLock._create_lock_dir') + @mock.patch('crmsh.join_lock.JoinLock.join_timeout', new_callable=mock.PropertyMock) + @mock.patch('time.time') + def test_lock_or_wait_break(self, mock_time, mock_time_out, mock_create): + mock_time.return_value = 10000 + mock_time_out.return_value = 120 + mock_create.return_value = True + + self.lock_inst._lock_or_wait() + self.assertEqual(self.lock_inst.lock_owner, True) + + mock_time.assert_called_once_with() + mock_time_out.assert_called_once_with() + + @mock.patch('time.sleep') + @mock.patch('crmsh.bootstrap.warn') + @mock.patch('crmsh.join_lock.JoinLock._get_online_nodelist') + @mock.patch('crmsh.join_lock.JoinLock._create_lock_dir') + @mock.patch('crmsh.join_lock.JoinLock.join_timeout', new_callable=mock.PropertyMock) + @mock.patch('time.time') + def test_lock_or_wait_timed_out(self, mock_time, mock_time_out, mock_create, + mock_get_nodelist, mock_warn, mock_sleep): + mock_time.side_effect = [10000, 10120, 10500] + mock_time_out.side_effect = [120, 120, 120] + mock_create.side_effect = [False, False] + mock_get_nodelist.side_effect = ["node1", "node1"] + + with self.assertRaises(TimeoutError) as err: + self.lock_inst._lock_or_wait() + self.assertEqual("Join process failed after 120 seconds. Cannot continue since the lock directory exists at the init node (node1:/tmp/.crmsh_join_lock_directory)", str(err.exception)) + + mock_time.assert_has_calls([mock.call(), mock.call(), mock.call()]) + mock_time_out.assert_has_calls([mock.call(), mock.call(), mock.call()]) + mock_create.assert_has_calls([mock.call(), mock.call()]) + mock_get_nodelist.assert_has_calls([mock.call(), mock.call()]) + mock_warn.assert_called_once_with("Other node still joining, wait at most 120s...") + mock_sleep.assert_has_calls([mock.call(10), mock.call(10)]) + + @mock.patch('time.sleep') + @mock.patch('crmsh.bootstrap.warn') + @mock.patch('crmsh.join_lock.JoinLock._get_online_nodelist') + @mock.patch('crmsh.join_lock.JoinLock._create_lock_dir') + @mock.patch('crmsh.join_lock.JoinLock.join_timeout', new_callable=mock.PropertyMock) + @mock.patch('time.time') + def test_lock_or_wait_again(self, mock_time, mock_time_out, mock_create, + mock_get_nodelist, mock_warn, mock_sleep): + mock_time.side_effect = [10000, 10010, 10020] + mock_time_out.side_effect = [120, 120, 120] + mock_create.side_effect = [False, False, True] + mock_get_nodelist.side_effect = [["node1"], ["node1", "node2"]] + + self.lock_inst._lock_or_wait() + + mock_time.assert_has_calls([mock.call(), mock.call(), mock.call()]) + mock_time_out.assert_has_calls([mock.call(), mock.call(), mock.call()]) + mock_create.assert_has_calls([mock.call(), mock.call(), mock.call()]) + mock_get_nodelist.assert_has_calls([mock.call(), mock.call()]) + mock_warn.assert_called_once_with("Other node still joining, wait at most 120s...") + mock_sleep.assert_called_once_with(10) + + @mock.patch('crmsh.join_lock.JoinLock.unlock') + @mock.patch('crmsh.join_lock.JoinLock._lock_or_wait') + def test_lock_exception(self, mock_wait, mock_unlock): + with self.assertRaises(ValueError): + with self.lock_inst.lock(): + raise ValueError + mock_wait.assert_called_once_with() + mock_unlock.assert_called_once_with() + + @mock.patch('crmsh.join_lock.bootstrap.error') + @mock.patch('crmsh.join_lock.JoinLock.unlock') + @mock.patch('crmsh.join_lock.JoinLock._lock_or_wait') + def test_lock_ssh_error(self, mock_wait, mock_unlock, mock_error): + mock_wait.side_effect = join_lock.SSHError("ssh error") + mock_error.side_effect = SystemExit + + with self.assertRaises(SystemExit): + with self.lock_inst.lock(): + pass + + mock_error.assert_called_once_with("ssh error") + mock_wait.assert_called_once_with() + mock_unlock.assert_called_once_with() + + @mock.patch('crmsh.join_lock.JoinLock.unlock') + @mock.patch('crmsh.join_lock.JoinLock._lock_or_wait') + def test_lock(self, mock_wait, mock_unlock): + with self.lock_inst.lock(): + pass + mock_wait.assert_called_once_with() + mock_unlock.assert_called_once_with() + + @mock.patch('crmsh.join_lock.JoinLock._run') + def test_unlock(self, mock_run): + self.lock_inst.lock_owner = True + self.lock_inst.unlock() + mock_run.assert_called_once_with(join_lock.JoinLock.RM_CMD) _______________________________________________ openSUSE Commits mailing list -- [email protected] To unsubscribe, email [email protected] List Netiquette: https://en.opensuse.org/openSUSE:Mailing_list_netiquette List Archives: https://lists.opensuse.org/archives/list/[email protected]
