Repository: ambari Updated Branches: refs/heads/branch-2.4 4146ab9ce -> 30a774f56 refs/heads/branch-2.5 372ffadaa -> 9e8c48504 refs/heads/trunk 22a4875cc -> 426468881
AMBARI-18832. Ambari can autoformat NameNode in a production cluster (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/42646888 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/42646888 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/42646888 Branch: refs/heads/trunk Commit: 42646888197148f67ae339568cd7d1bf56000236 Parents: 22a4875 Author: Andrew Onishuk <[email protected]> Authored: Thu Nov 10 10:53:44 2016 +0200 Committer: Andrew Onishuk <[email protected]> Committed: Thu Nov 10 10:53:44 2016 +0200 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 22 +++++++++++++------- .../python/stacks/2.0.6/HDFS/test_namenode.py | 10 ++++----- 2 files changed, 20 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/42646888/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index b7735f4..23119f0 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -307,7 +307,7 @@ def format_namenode(force=None): conf_dir=hadoop_conf_dir) else: if not is_namenode_formatted(params): - Execute(format("yes Y | hdfs --config {hadoop_conf_dir} namenode -format"), + Execute(format("hdfs --config {hadoop_conf_dir} namenode -format -nonInteractive"), user = params.hdfs_user, path = [params.hadoop_bin_dir] ) @@ -328,7 +328,7 @@ def format_namenode(force=None): nn_name_dirs = params.dfs_name_dir.split(',') if not is_namenode_formatted(params): try: - Execute(format("yes Y | hdfs --config {hadoop_conf_dir} namenode -format"), + Execute(format("hdfs --config {hadoop_conf_dir} namenode -format -nonInteractive"), user = params.hdfs_user, path = [params.hadoop_bin_dir] ) @@ -384,18 +384,26 @@ def is_namenode_formatted(params): ) marked = True + if marked: + return True + # Check if name dirs are not empty for name_dir in nn_name_dirs: + code, out = shell.call(("ls", name_dir)) + dir_exists_and_valid = bool(not code) + + if not dir_exists_and_valid: # situations if disk exists but is crashed at the moment (ls: reading directory ...: Input/output error) + Logger.info(format("NameNode will not be formatted because the directory {name_dir} is missing or cannot be checked for content. {out}")) + return True + try: Execute(format("ls {name_dir} | wc -l | grep -q ^0$"), ) - marked = False except Fail: - marked = True - Logger.info(format("ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs {nn_name_dirs}")) - break + Logger.info(format("NameNode will not be formatted since {name_dir} exists and contains content")) + return True - return marked + return False @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def decommission(): http://git-wip-us.apache.org/repos/asf/ambari/blob/42646888/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py index 7d1d4f1..2627ddf 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py @@ -57,7 +57,7 @@ class TestNamenode(RMFTestCase): ) self.assert_configure_default() self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l | grep -q ^0$',) - self.assertResourceCalled('Execute', 'yes Y | hdfs --config /etc/hadoop/conf namenode -format', + self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf namenode -format -nonInteractive', path = ['/usr/bin'], user = 'hdfs', ) @@ -172,7 +172,7 @@ class TestNamenode(RMFTestCase): ) self.assert_configure_default() self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l | grep -q ^0$',) - self.assertResourceCalled('Execute', 'yes Y | hdfs --config /etc/hadoop/conf namenode -format', + self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf namenode -format -nonInteractive', path = ['/usr/bin'], user = 'hdfs', ) @@ -300,7 +300,7 @@ class TestNamenode(RMFTestCase): ) self.assert_configure_secured() self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l | grep -q ^0$',) - self.assertResourceCalled('Execute', 'yes Y | hdfs --config /etc/hadoop/conf namenode -format', + self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf namenode -format -nonInteractive', path = ['/usr/bin'], user = 'hdfs', ) @@ -725,7 +725,7 @@ class TestNamenode(RMFTestCase): # verify that active namenode was formatted self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l | grep -q ^0$',) - self.assertResourceCalled('Execute', 'yes Y | hdfs --config /etc/hadoop/conf namenode -format', + self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf namenode -format -nonInteractive', path = ['/usr/bin'], user = 'hdfs', ) @@ -1776,7 +1776,7 @@ class TestNamenode(RMFTestCase): config_dict = json_content, stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES, - call_mocks = [(0, None, ''), (0, None)], + call_mocks = [(0, None), (0, None, ''), (0, None)], mocks_dict = mocks_dict) # jump right to the start of the NN and then verify that we DO NOT call HdfsResource after
