Repository: ambari Updated Branches: refs/heads/trunk 25191edf3 -> b6c115ba2
AMBARI-11605. Restarting HistoryServer fails during RU because NameNode is in safemode (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b6c115ba Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b6c115ba Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b6c115ba Branch: refs/heads/trunk Commit: b6c115ba2487d16101e253f034e91ae686edb755 Parents: 25191ed Author: Alejandro Fernandez <[email protected]> Authored: Sat May 30 14:27:18 2015 -0700 Committer: Alejandro Fernandez <[email protected]> Committed: Tue Jun 2 10:18:06 2015 -0700 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 78 ++++++++++++++------ .../HDFS/2.1.0.2.0/package/scripts/namenode.py | 23 ++++-- .../HDFS/2.1.0.2.0/package/scripts/utils.py | 21 ++++-- .../python/stacks/2.0.6/HDFS/test_namenode.py | 56 ++++---------- 4 files changed, 95 insertions(+), 83 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c115ba/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index d8666b3..2ab3178 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -18,12 +18,24 @@ limitations under the License. """ import os.path -from resource_management import * -from resource_management.core.logger import Logger -from resource_management.core.exceptions import ComponentIsNotRunning + +from resource_management.core import shell +from resource_management.core.source import Template +from resource_management.core.resources.system import File, Execute, Directory +from resource_management.core.resources.service import Service +from resource_management.libraries.functions.format import format from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop +from ambari_commons import OSCheck, OSConst from ambari_commons.os_family_impl import OsFamilyImpl, OsFamilyFuncImpl -from ambari_commons import OSConst + +if OSCheck.is_windows_family(): + from resource_management.libraries.functions.windows_service_utils import check_windows_service_status + +from resource_management.core.shell import as_user +from resource_management.core.exceptions import Fail +from resource_management.core.logger import Logger + from utils import service, safe_zkfc_op from setup_ranger_hdfs import setup_ranger_hdfs @@ -77,24 +89,40 @@ def namenode(action=None, do_format=True, rolling_restart=False, env=None): Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"), user = params.hdfs_user) + is_namenode_safe_mode_off = format("hadoop dfsadmin -fs {namenode_address} -safemode get | grep 'Safe mode is OFF'") + is_active_namenode_cmd = as_user(format("hdfs --config {hadoop_conf_dir} haadmin -getServiceState {namenode_id} | grep active"), params.hdfs_user, env={'PATH':params.hadoop_bin_dir}) + + # During normal operations, if HA is enabled and it is in standby, then stay in current state, otherwise, leave safemode. + # During Rolling Upgrade, both namenodes must leave safemode. + + # ___Scenario_________|_Expected safemode state___|_Wait for safemode OFF____| + # 1 (HA and active) | ON -> OFF | Yes | + # 2 (HA and standby) | no change (yes during RU) | no check (yes during RU) | + # 3 (no-HA) | ON -> OFF | Yes | + leave_safe_mode = False + msg = "" if params.dfs_ha_enabled: - dfs_check_nn_status_cmd = as_user(format("hdfs --config {hadoop_conf_dir} haadmin -getServiceState {namenode_id} | grep active"), params.hdfs_user, env={'PATH':params.hadoop_bin_dir}) + code, out = shell.call(is_active_namenode_cmd, logoutput=True) # If active NN, code will be 0 + if code == 0: # active + leave_safe_mode = True + msg = "Must leave safemode since High Availability is enabled and this is the Active NameNode." + elif rolling_restart: + leave_safe_mode = True + msg = "Must leave safemode since High Availability is enabled during a Rolling Upgrade" else: - dfs_check_nn_status_cmd = None + msg = "Must leave safemode since High Availability is not enabled." + leave_safe_mode = True - namenode_safe_mode_off = format("hadoop dfsadmin -fs {namenode_address} -safemode get | grep 'Safe mode is OFF'") - - # If HA is enabled and it is in standby, then stay in safemode, otherwise, leave safemode. - leave_safe_mode = True - if dfs_check_nn_status_cmd is not None: - code, out = shell.call(dfs_check_nn_status_cmd) # If active NN, code will be 0 - if code != 0: - leave_safe_mode = False + if not msg: + msg = "Will remain in the current safemode state." + Logger.info(msg) if leave_safe_mode: # First check if Namenode is not in 'safemode OFF' (equivalent to safemode ON), if so, then leave it - code, out = shell.call(namenode_safe_mode_off, user=params.hdfs_user) + Logger.info("Checking the NameNode safemode status since may need to transition from ON to OFF.") + code, out = shell.call(is_namenode_safe_mode_off, user=params.hdfs_user) if code != 0: + Logger.info("Will need to leave safemode, state should be OFF.") leave_safe_mode_cmd = format("hdfs --config {hadoop_conf_dir} dfsadmin -fs {namenode_address} -safemode leave") Execute(leave_safe_mode_cmd, tries=10, @@ -103,15 +131,17 @@ def namenode(action=None, do_format=True, rolling_restart=False, env=None): path=[params.hadoop_bin_dir], ) - # Verify if Namenode should be in safemode OFF - Execute(namenode_safe_mode_off, - tries=40, - try_sleep=10, - path=[params.hadoop_bin_dir], - user=params.hdfs_user, - only_if=dfs_check_nn_status_cmd #skip when HA not active - ) - create_hdfs_directories(dfs_check_nn_status_cmd) + Logger.info("Checking if safemode state is now OFF.") + # Verify if Namenode should be in safemode OFF + Execute(is_namenode_safe_mode_off, + tries=40, + try_sleep=10, + path=[params.hadoop_bin_dir], + user=params.hdfs_user + ) + + # Always run on this non-HA, or active NameNode during HA. + create_hdfs_directories(is_active_namenode_cmd) elif action == "stop": import params service( http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c115ba/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py index e557317..3494ac9 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode.py @@ -19,27 +19,34 @@ limitations under the License. import sys import os +import json +import tempfile +from datetime import datetime import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. -import tempfile -from resource_management import * + +from resource_management import Script +from resource_management.core.resources.system import Execute from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import hdp_select +from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version +from resource_management.libraries.functions.format import format from resource_management.libraries.functions.security_commons import build_expectations, \ cached_kinit_executor, get_params_from_filesystem, validate_security_config_properties, \ FILE_TYPE_XML -from resource_management.libraries.functions.version import compare_versions, \ - format_hdp_stack_version -from resource_management.libraries.functions.format import format + from resource_management.core.exceptions import Fail -from datetime import datetime +from resource_management.core.shell import as_user +from resource_management.core.logger import Logger + +from ambari_commons.os_family_impl import OsFamilyImpl +from ambari_commons import OSConst import namenode_upgrade from hdfs_namenode import namenode from hdfs import hdfs import hdfs_rebalance from utils import failover_namenode -from ambari_commons.os_family_impl import OsFamilyImpl -from ambari_commons import OSConst + # hashlib is supplied as of Python 2.5 as the replacement interface for md5 # and other secure hashes. In 2.6, md5 is deprecated. Import hashlib if http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c115ba/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py index 59a7b97..11076aa 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py @@ -21,10 +21,14 @@ import re import urllib2 import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. -from resource_management import * +from resource_management.core.resources.system import Directory, File, Execute from resource_management.libraries.functions.format import format -from resource_management.core.shell import call, checked_call +from resource_management.libraries.functions import check_process_status +from resource_management.libraries.functions.version import compare_versions +from resource_management.core import shell +from resource_management.core.shell import as_user, as_sudo from resource_management.core.exceptions import ComponentIsNotRunning +from resource_management.core.logger import Logger from zkfc_slave import ZkfcSlave @@ -34,6 +38,7 @@ def safe_zkfc_op(action, env): :param action: start or stop :param env: environment """ + Logger.info("Performing action {0} on zkfc.".format(action)) zkfc = None if action == "start": try: @@ -60,7 +65,7 @@ def failover_namenode(): """ import params check_service_cmd = format("hdfs haadmin -getServiceState {namenode_id}") - code, out = call(check_service_cmd, logoutput=True, user=params.hdfs_user) + code, out = shell.call(check_service_cmd, logoutput=True, user=params.hdfs_user) state = "unknown" if code == 0 and out: @@ -78,22 +83,22 @@ def failover_namenode(): check_standby_cmd = format("hdfs haadmin -getServiceState {namenode_id} | grep standby") # process may already be down. try one time, then proceed - code, out = call(check_standby_cmd, user=params.hdfs_user, logoutput=True) + code, out = shell.call(check_standby_cmd, user=params.hdfs_user, logoutput=True) Logger.info(format("Rolling Upgrade - check for standby returned {code}")) if code == 255 and out: - Logger.info("Rolling Upgrade - namenode is already down") + Logger.info("Rolling Upgrade - namenode is already down.") else: if was_zkfc_killed: # Only mandate that this be the standby namenode if ZKFC was indeed killed to initiate a failover. + Logger.info("Waiting for this NameNode to become the standby one.") Execute(check_standby_cmd, user=params.hdfs_user, tries=50, try_sleep=6, logoutput=True) - else: - Logger.info("Rolling Upgrade - Host %s is the standby namenode." % str(params.hostname)) + Logger.info("Rolling Upgrade - Host %s is already the standby namenode." % str(params.hostname)) def kill_zkfc(zkfc_user): @@ -109,7 +114,7 @@ def kill_zkfc(zkfc_user): zkfc_pid_file = get_service_pid_file("zkfc", zkfc_user) if zkfc_pid_file: check_process = format("ls {zkfc_pid_file} > /dev/null 2>&1 && ps -p `cat {zkfc_pid_file}` > /dev/null 2>&1") - code, out = call(check_process) + code, out = shell.call(check_process) if code == 0: Logger.debug("ZKFC is running and will be killed to initiate namenode failover.") kill_command = format("{check_process} && kill -9 `cat {zkfc_pid_file}` > /dev/null 2>&1") http://git-wip-us.apache.org/repos/asf/ambari/blob/b6c115ba/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py index 68229d2..e9f7829 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py @@ -96,13 +96,12 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://c6405.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'", path = ['/usr/bin'], tries = 40, - only_if = None, user = 'hdfs', try_sleep = 10, ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = False, - only_if = None, + only_if="ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState None | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'wasb://[email protected]', @@ -118,7 +117,7 @@ class TestNamenode(RMFTestCase): ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', security_enabled = False, - only_if = None, + only_if="ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState None | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'wasb://[email protected]', @@ -134,7 +133,7 @@ class TestNamenode(RMFTestCase): ) self.assertResourceCalled('HdfsResource', None, security_enabled = False, - only_if = None, + only_if="ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState None | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'wasb://[email protected]', @@ -213,13 +212,12 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'", path = ['/usr/bin'], tries = 40, - only_if = None, user = 'hdfs', try_sleep = 10, ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = False, - only_if = None, + only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState None | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://c6401.ambari.apache.org:8020', @@ -235,7 +233,7 @@ class TestNamenode(RMFTestCase): ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', security_enabled = False, - only_if = None, + only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState None | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://c6401.ambari.apache.org:8020', @@ -251,7 +249,7 @@ class TestNamenode(RMFTestCase): ) self.assertResourceCalled('HdfsResource', None, security_enabled = False, - only_if = None, + only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState None | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://c6401.ambari.apache.org:8020', @@ -353,7 +351,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'", path = ['/usr/bin'], tries = 40, - only_if = None, user = 'hdfs', try_sleep = 10, ) @@ -368,7 +365,7 @@ class TestNamenode(RMFTestCase): type = 'directory', action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0777, - only_if = None, + only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState None | grep active'" ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', security_enabled = True, @@ -381,11 +378,11 @@ class TestNamenode(RMFTestCase): type = 'directory', action = ['create_on_execute'], hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', mode = 0770, - only_if = None, + only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState None | grep active'" ) self.assertResourceCalled('HdfsResource', None, security_enabled = True, - only_if = None, + only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState None | grep active'", keytab = '/etc/security/keytabs/hdfs.headless.keytab', hadoop_bin_dir = '/usr/bin', kinit_path_local = '/usr/bin/kinit', @@ -451,13 +448,6 @@ class TestNamenode(RMFTestCase): environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'}, not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1', ) - self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'", - path = ['/usr/bin'], - tries = 40, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'", - user = 'hdfs', - try_sleep = 10, - ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = False, only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'", @@ -543,13 +533,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs', user = 'hdfs', ) - self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'", - path = ['/usr/bin'], - tries = 40, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'", - user = 'hdfs', - try_sleep = 10, - ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = True, only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'", @@ -644,13 +627,6 @@ class TestNamenode(RMFTestCase): environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'}, not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1', ) - self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'", - path = ['/usr/bin'], - tries = 40, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'", - user = 'hdfs', - try_sleep = 10, - ) self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = False, only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'", @@ -742,13 +718,7 @@ class TestNamenode(RMFTestCase): environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'}, not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1', ) - self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'", - path = ['/usr/bin'], - tries = 40, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn2 | grep active'", - user = 'hdfs', - try_sleep = 10, - ) + self.assertResourceCalled('HdfsResource', '/tmp', security_enabled = False, only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn2 | grep active'", @@ -1300,7 +1270,7 @@ class TestNamenode(RMFTestCase): ) self.assertNoMoreResources() - @patch("resource_management.core.shell.call") + @patch.object(shell, "call") def test_pre_rolling_restart_21_and_lower_params(self, call_mock): config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/nn_ru_lzo.json" with open(config_file, "r") as f: @@ -1323,7 +1293,7 @@ class TestNamenode(RMFTestCase): self.assertEquals("/usr/bin", sys.modules["params"].hadoop_bin_dir) self.assertEquals("/usr/lib/hadoop/sbin", sys.modules["params"].hadoop_bin) - @patch("resource_management.core.shell.call") + @patch.object(shell, "call") def test_pre_rolling_restart_22_params(self, call_mock): config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/nn_ru_lzo.json" with open(config_file, "r") as f: @@ -1348,7 +1318,7 @@ class TestNamenode(RMFTestCase): self.assertEquals("/usr/hdp/current/hadoop-client/bin", sys.modules["params"].hadoop_bin_dir) self.assertEquals("/usr/hdp/current/hadoop-client/sbin", sys.modules["params"].hadoop_bin) - @patch("resource_management.core.shell.call") + @patch.object(shell, "call") def test_pre_rolling_restart_23_params(self, call_mock): import itertools
