Repository: ambari Updated Branches: refs/heads/trunk 126626857 -> 35ebd80d8
AMBARI-16162. Reduce NN start time by removing redundant haadmin calls. (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/35ebd80d Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/35ebd80d Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/35ebd80d Branch: refs/heads/trunk Commit: 35ebd80d87e2a46c1dfcccc3a243bf59fb3c510c Parents: 1266268 Author: Andrew Onishuk <[email protected]> Authored: Thu Apr 28 18:26:49 2016 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Thu Apr 28 18:26:49 2016 +0300 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 19 ++++++----- .../package/scripts/setup_ranger_hdfs.py | 6 ++-- .../python/stacks/2.0.6/HDFS/test_namenode.py | 36 ++------------------ 3 files changed, 16 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/35ebd80d/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index 357eb71..635f159 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -180,6 +180,7 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, # EU non-HA | no change | no check | check_for_safemode_off = False + is_active_namenode = False msg = "" if params.dfs_ha_enabled: if upgrade_type is not None: @@ -187,14 +188,16 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, msg = "Must wait to leave safemode since High Availability is enabled during a Stack Upgrade" else: Logger.info("Wait for NameNode to become active.") - if is_active_namenode(hdfs_binary): # active + if check_is_active_namenode(hdfs_binary): # active check_for_safemode_off = True + is_active_namenode = True msg = "Must wait to leave safemode since High Availability is enabled and this is the Active NameNode." else: msg = "Will remain in the current safemode state." else: msg = "Must wait to leave safemode since High Availability is not enabled." check_for_safemode_off = True + is_active_namenode = True Logger.info(msg) @@ -209,8 +212,11 @@ def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, wait_for_safemode_off(hdfs_binary) # Always run this on non-HA, or active NameNode during HA. - create_hdfs_directories(is_active_namenode_cmd) - create_ranger_audit_hdfs_directories(is_active_namenode_cmd) + if is_active_namenode: + create_hdfs_directories() + create_ranger_audit_hdfs_directories() + else: + Logger.info("Skipping creating hdfs directories as is not active NN.") elif action == "stop": import params @@ -267,7 +273,7 @@ def create_name_dirs(directories): ) -def create_hdfs_directories(check): +def create_hdfs_directories(): import params params.HdfsResource(params.hdfs_tmp_dir, @@ -275,18 +281,15 @@ def create_hdfs_directories(check): action="create_on_execute", owner=params.hdfs_user, mode=0777, - only_if=check ) params.HdfsResource(params.smoke_hdfs_user_dir, type="directory", action="create_on_execute", owner=params.smoke_user, mode=params.smoke_hdfs_user_mode, - only_if=check ) params.HdfsResource(None, action="execute", - only_if=check #skip creation when HA not active ) def format_namenode(force=None): @@ -508,7 +511,7 @@ def is_namenode_bootstrapped(params): return marked -def is_active_namenode(hdfs_binary): +def check_is_active_namenode(hdfs_binary): """ Checks if current NameNode is active. Waits up to 30 seconds. If other NameNode is active returns False. :return: True if current NameNode is active, False otherwise http://git-wip-us.apache.org/repos/asf/ambari/blob/35ebd80d/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py index 040c69c..72eb3ad 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/setup_ranger_hdfs.py @@ -95,7 +95,7 @@ def setup_ranger_hdfs(upgrade_type=None): else: Logger.info('Ranger admin not installed') -def create_ranger_audit_hdfs_directories(check): +def create_ranger_audit_hdfs_directories(): import params if params.has_ranger_admin: @@ -107,7 +107,6 @@ def create_ranger_audit_hdfs_directories(check): group=params.hdfs_user, mode=0755, recursive_chmod=True, - only_if=check ) params.HdfsResource("/ranger/audit/hdfs", type="directory", @@ -116,8 +115,7 @@ def create_ranger_audit_hdfs_directories(check): group=params.hdfs_user, mode=0700, recursive_chmod=True, - only_if=check ) - params.HdfsResource(None, action="execute", only_if=check) + params.HdfsResource(None, action="execute") else: Logger.info('Ranger admin not installed') http://git-wip-us.apache.org/repos/asf/ambari/blob/35ebd80d/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py index f56a0d2..34b639e 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py @@ -100,7 +100,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/tmp', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if=True, keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'wasb://[email protected]', @@ -118,7 +117,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/user/ambari-qa', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if=True, keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'wasb://[email protected]', @@ -136,7 +134,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', None, immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if=True, keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'wasb://[email protected]', @@ -218,7 +215,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/tmp', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = True, keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://c6401.ambari.apache.org:8020', @@ -236,7 +232,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/user/ambari-qa', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = True, keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://c6401.ambari.apache.org:8020', @@ -254,7 +249,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', None, immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = True, keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://c6401.ambari.apache.org:8020', @@ -361,8 +355,7 @@ class TestNamenode(RMFTestCase): hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', - mode = 0777, - only_if = True + mode = 0777 ) self.assertResourceCalled('HdfsResource', '/user/ambari-qa', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, @@ -376,13 +369,11 @@ class TestNamenode(RMFTestCase): hadoop_conf_dir = '/etc/hadoop/conf', type = 'directory', action = ['create_on_execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name='hdfs', default_fs='hdfs://c6401.ambari.apache.org:8020', - mode = 0770, - only_if = True + mode = 0770 ) self.assertResourceCalled('HdfsResource', None, immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = True, - only_if = True, keytab = '/etc/security/keytabs/hdfs.headless.keytab', hadoop_bin_dir = '/usr/bin', kinit_path_local = '/usr/bin/kinit', @@ -453,7 +444,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/tmp', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -471,7 +461,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/user/ambari-qa', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -489,7 +478,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', None, immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -555,7 +543,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/tmp', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -573,7 +560,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/user/ambari-qa', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -591,7 +577,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', None, immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -663,7 +648,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/tmp', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = True, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = '/etc/security/keytabs/hdfs.headless.keytab', hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -681,7 +665,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/user/ambari-qa', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = True, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = '/etc/security/keytabs/hdfs.headless.keytab', hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -699,7 +682,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', None, immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = True, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = '/etc/security/keytabs/hdfs.headless.keytab', hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -771,7 +753,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/tmp', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -789,7 +770,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/user/ambari-qa', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -807,7 +787,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', None, immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn1 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -881,7 +860,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/tmp', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -899,7 +877,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/user/ambari-qa', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -917,7 +894,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', None, immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -999,7 +975,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/tmp', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -1017,7 +992,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/user/ambari-qa', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -1035,7 +1009,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', None, immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -1115,7 +1088,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/tmp', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -1133,7 +1105,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', '/user/ambari-qa', immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -1151,7 +1122,6 @@ class TestNamenode(RMFTestCase): self.assertResourceCalled('HdfsResource', None, immutable_paths = self.DEFAULT_IMMUTABLE_PATHS, security_enabled = False, - only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -ns ns1 -getServiceState nn2 | grep active'", keytab = UnknownConfigurationMock(), hadoop_bin_dir = '/usr/bin', default_fs = 'hdfs://ns1', @@ -1592,7 +1562,7 @@ class TestNamenode(RMFTestCase): self.assertTrue(calls[0].startsWith("conf-select create-conf-dir --package hadoop --stack-version 2.3.2.0-2844 --conf-version 0")) - @patch("hdfs_namenode.is_active_namenode") + @patch("hdfs_namenode.check_is_active_namenode") @patch("resource_management.libraries.functions.setup_ranger_plugin_xml.setup_ranger_plugin") @patch("utils.get_namenode_states") def test_upgrade_restart_eu_with_ranger(self, get_namenode_states_mock, setup_ranger_plugin_mock, is_active_nn_mock):
