Repository: ambari Updated Branches: refs/heads/branch-2.2 f4e6e59dc -> 24188d3a0
AMBARI-16028: Namenode marked as INITIAL standby could potentially never start if other namenode is down (jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/24188d3a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/24188d3a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/24188d3a Branch: refs/heads/branch-2.2 Commit: 24188d3a09dace7e064210dfd6a70155c97ff2d0 Parents: f4e6e59 Author: Jayush Luniya <[email protected]> Authored: Fri Apr 22 11:42:17 2016 -0700 Committer: Jayush Luniya <[email protected]> Committed: Fri Apr 22 11:42:17 2016 -0700 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/hdfs_namenode.py | 41 +++++++++++++++++--- .../2.1.0.2.0/package/scripts/params_linux.py | 8 +++- .../python/stacks/2.0.6/HDFS/test_namenode.py | 6 +++ 3 files changed, 48 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/24188d3a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py index b62c3c5..a50a3ba 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py @@ -450,11 +450,11 @@ def decommission(): def bootstrap_standby_namenode(params, use_path=False): - + mark_dirs = params.namenode_bootstrapped_mark_dirs bin_path = os.path.join(params.hadoop_bin_dir, '') if use_path else "" - try: iterations = 50 + bootstrapped = False bootstrap_cmd = format("{bin_path}hdfs namenode -bootstrapStandby -nonInteractive") # Blue print based deployments start both NN in parallel and occasionally # the first attempt to bootstrap may fail. Depending on how it fails the @@ -462,22 +462,53 @@ def bootstrap_standby_namenode(params, use_path=False): # bootstrap succeeded). The solution is to call with -force option but only # during initial start if params.command_phase == "INITIAL_START": + # force bootstrap in INITIAL_START phase bootstrap_cmd = format("{bin_path}hdfs namenode -bootstrapStandby -nonInteractive -force") + elif is_namenode_bootstrapped(params): + # Once out of INITIAL_START phase bootstrap only if we couldnt bootstrap during cluster deployment + return True Logger.info("Boostrapping standby namenode: %s" % (bootstrap_cmd)) for i in range(iterations): Logger.info('Try %d out of %d' % (i+1, iterations)) code, out = shell.call(bootstrap_cmd, logoutput=False, user=params.hdfs_user) if code == 0: Logger.info("Standby namenode bootstrapped successfully") - return True + bootstrapped = True + break elif code == 5: Logger.info("Standby namenode already bootstrapped") - return True + bootstrapped = True + break else: Logger.warning('Bootstrap standby namenode failed with %d error code. Will retry' % (code)) except Exception as ex: Logger.error('Bootstrap standby namenode threw an exception. Reason %s' %(str(ex))) - return False + if bootstrapped: + for mark_dir in mark_dirs: + Directory(mark_dir, + recursive = True + ) + return bootstrapped + +def is_namenode_bootstrapped(params): + mark_dirs = params.namenode_bootstrapped_mark_dirs + nn_name_dirs = params.dfs_name_dir.split(',') + marked = False + # Check if name directories have been marked as formatted + for mark_dir in mark_dirs: + if os.path.isdir(mark_dir): + marked = True + Logger.info(format("{mark_dir} exists. Standby Namenode already bootstrapped")) + break + + # Ensure that all mark dirs created for all name directories + if marked: + for mark_dir in mark_dirs: + Directory(mark_dir, + recursive = True + ) + + return marked def is_active_namenode(hdfs_binary): http://git-wip-us.apache.org/repos/asf/ambari/blob/24188d3a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py index 4347abb..af619ac 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py @@ -227,14 +227,18 @@ smoke_hdfs_user_mode = 0770 hdfs_namenode_format_disabled = default("/configurations/cluster-env/hdfs_namenode_format_disabled", False) hdfs_namenode_formatted_mark_suffix = "/namenode-formatted/" +hdfs_namenode_bootstrapped_mark_suffix = "/namenode-bootstrapped/" namenode_formatted_old_mark_dirs = ["/var/run/hadoop/hdfs/namenode-formatted", format("{hadoop_pid_dir_prefix}/hdfs/namenode/formatted"), "/var/lib/hdfs/namenode/formatted"] dfs_name_dirs = dfs_name_dir.split(",") namenode_formatted_mark_dirs = [] +namenode_bootstrapped_mark_dirs = [] for dn_dir in dfs_name_dirs: - tmp_mark_dir = format("{dn_dir}{hdfs_namenode_formatted_mark_suffix}") - namenode_formatted_mark_dirs.append(tmp_mark_dir) + tmp_format_mark_dir = format("{dn_dir}{hdfs_namenode_formatted_mark_suffix}") + tmp_bootstrap_mark_dir = format("{dn_dir}{hdfs_namenode_bootstrapped_mark_suffix}") + namenode_formatted_mark_dirs.append(tmp_format_mark_dir) + namenode_bootstrapped_mark_dirs.append(tmp_bootstrap_mark_dir) # Use the namenode RPC address if configured, otherwise, fallback to the default file system namenode_address = None http://git-wip-us.apache.org/repos/asf/ambari/blob/24188d3a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py index d6e32e8..f9e8a4d 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py @@ -828,6 +828,9 @@ class TestNamenode(RMFTestCase): content = Template('exclude_hosts_list.j2'), group = 'hadoop', ) + self.assertResourceCalled('Directory', '/hadoop/hdfs/namenode/namenode-bootstrapped/', + recursive = True + ) self.assertResourceCalled('Directory', '/var/run/hadoop', owner = 'hdfs', group = 'hadoop', @@ -941,6 +944,9 @@ class TestNamenode(RMFTestCase): content = Template('exclude_hosts_list.j2'), group = 'hadoop', ) + self.assertResourceCalled('Directory', '/hadoop/hdfs/namenode/namenode-bootstrapped/', + recursive = True + ) self.assertResourceCalled('Directory', '/var/run/hadoop', owner = 'hdfs', group = 'hadoop',
