Repository: ambari Updated Branches: refs/heads/trunk eba63884b -> e4bac844b
AMBARI-19830. HDP 3.0 TP - Support changed configs and scripts for HDFS (alejandro) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e4bac844 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e4bac844 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e4bac844 Branch: refs/heads/trunk Commit: e4bac844b33539b06e4208e058e7d89c7761eb99 Parents: eba6388 Author: Alejandro Fernandez <afernan...@hortonworks.com> Authored: Thu Jan 26 17:58:13 2017 -0800 Committer: Alejandro Fernandez <afernan...@hortonworks.com> Committed: Fri Feb 3 15:47:13 2017 -0800 ---------------------------------------------------------------------- .../HDFS/3.0.0.3.0/configuration/hdfs-site.xml | 4 + .../3.0.0.3.0/package/scripts/params_linux.py | 2 +- .../HDFS/3.0.0.3.0/package/scripts/utils.py | 8 +- .../stacks/2.5/HIVE/test_hive_server_int.py | 2 +- ambari-server/src/test/python/unitTests.py | 106 +++++++++++++++++-- 5 files changed, 106 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e4bac844/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hdfs-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hdfs-site.xml b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hdfs-site.xml index 689b6d08..60fde60 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hdfs-site.xml +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/configuration/hdfs-site.xml @@ -35,13 +35,17 @@ </value-attributes> <on-ambari-upgrade add="true"/> </property> + <!-- + This property is deleted in Hadoop 3.0. Need to remove it during Stack Upgrade. <property> <name>dfs.support.append</name> <value>true</value> <description>to enable dfs append</description> <final>true</final> <on-ambari-upgrade add="true"/> + <deleted>true</deleted> </property> + --> <property> <name>dfs.webhdfs.enabled</name> <value>true</value> http://git-wip-us.apache.org/repos/asf/ambari/blob/e4bac844/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py index f7aa4c9..62a5edd 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/params_linux.py @@ -96,7 +96,7 @@ namenode_backup_dir = default("/configurations/hadoop-env/namenode_backup_dir", # hadoop default parameters mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*" hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") -hadoop_bin = stack_select.get_hadoop_dir("sbin") +hadoop_bin = stack_select.get_hadoop_dir("bin") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_home = stack_select.get_hadoop_dir("home") hadoop_secure_dn_user = hdfs_user http://git-wip-us.apache.org/repos/asf/ambari/blob/e4bac844/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/utils.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/utils.py b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/utils.py index 48f5a1f..ab4308c 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/utils.py +++ b/ambari-server/src/main/resources/common-services/HDFS/3.0.0.3.0/package/scripts/utils.py @@ -250,19 +250,19 @@ def service(action=None, name=None, user=None, options="", create_pid_dir=False, except ComponentIsNotRunning: pass - hadoop_daemon = format("{hadoop_bin}/hadoop-daemon.sh") + hdfs_bin = format("{hadoop_bin}/hdfs") if user == "root": - cmd = [hadoop_daemon, "--config", params.hadoop_conf_dir, action, name] + cmd = [hdfs_bin, "--config", params.hadoop_conf_dir, "--daemon", action, name] if options: cmd += [options, ] daemon_cmd = as_sudo(cmd) else: - cmd = format("{ulimit_cmd} {hadoop_daemon} --config {hadoop_conf_dir} {action} {name}") + cmd = format("{ulimit_cmd} {hdfs_bin} --config {hadoop_conf_dir} --daemon {action} {name}") if options: cmd += " " + options daemon_cmd = as_user(cmd, user) - + if action == "start": # remove pid file from dead process File(pid_file, action="delete", not_if=process_id_exists_command) http://git-wip-us.apache.org/repos/asf/ambari/blob/e4bac844/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py index fb97612..6f53762 100644 --- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py @@ -243,7 +243,7 @@ class TestHiveServerInteractive(RMFTestCase): self.assertNoMoreResources() ''' - restart should not call slider destroy + #restart should not call slider destroy ''' @patch("os.path.isfile") @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") http://git-wip-us.apache.org/repos/asf/ambari/blob/e4bac844/ambari-server/src/test/python/unitTests.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/unitTests.py b/ambari-server/src/test/python/unitTests.py index 7941ed3..a8e72e9 100644 --- a/ambari-server/src/test/python/unitTests.py +++ b/ambari-server/src/test/python/unitTests.py @@ -20,6 +20,7 @@ import unittest import multiprocessing import os import sys +import re import traceback from Queue import Empty from random import shuffle @@ -86,6 +87,99 @@ def get_stack_name(): def get_stack_name(): return "HDP" +def extract_extends_field_from_file(metainfo): + pattern = re.compile(r"^\s*?<extends>(.*?)</extends>", re.IGNORECASE) + if os.path.isfile(metainfo): + with open(metainfo) as f: + for line in f.readlines(): + m = pattern.match(line) + if m and len(m.groups()) == 1: + extract = m.groups(1)[0] + return extract + return None + +def get_extends_field_from_metainfo(service_metainfo): + """ + Parse the metainfo.xml file to retrieve the <extends> value. + + @param service_metainfo: Path to the service metainfo.xml file + :return Extract the "extends" field if it exists and return its value. Otherwise, return None. + """ + extract = extract_extends_field_from_file(service_metainfo) + if extract is not None: + return extract + + # If couldn't find it in the service's metainfo.xml, check the stack's metainfo.xml + stack_metainfo = os.path.join(os.path.dirname(service_metainfo), "..", "..", "metainfo.xml") + extract = extract_extends_field_from_file(stack_metainfo) + + return extract + +def resolve_paths_to_import_from_common_services(metainfo_file, base_stack_folder, common_services_parent_dir, service): + """ + Get a list of paths to append to sys.path in order to import all of the needed modules. + This is important when a service has multiple definitions in common-services so that we import the correct one + instead of a higher version. + + @param metainfo_file: Path to the metainfo.xml file. + @param base_stack_folder: Path to stacks folder that does not include the version number. This can potentially be None. + @param common_services_parent_dir: Path to the common-services directory for a specified service, not including the version. + @param service: Service name + :return A list of paths to insert to sys.path by following the chain of inheritence. + """ + paths_to_import = [] + + if metainfo_file is None or service is None: + return paths_to_import + + # This could be either a version number of a path to common-services + extract = get_extends_field_from_metainfo(metainfo_file) + if extract is not None: + if "common-services" in extract: + # If in common-services, we are done. + scripts_path = os.path.join(common_services_parent_dir, extract, "package", "scripts") + paths_to_import.append(scripts_path) + else: + # If a version number, we need to import it and check that version as well. + inherited_from_older_version_path = os.path.join(base_stack_folder, "..", extract) + + metainfo_file = os.path.join(inherited_from_older_version_path, "services", service, "metainfo.xml") + if os.path.isdir(inherited_from_older_version_path): + paths_to_import += resolve_paths_to_import_from_common_services(metainfo_file, inherited_from_older_version_path, common_services_parent_dir, service) + else: + print "Service %s. Could not get extract <extends></extends> from metainfo file: %s. This may prevent modules from being imported." % (service, str(metainfo_file)) + + return paths_to_import + +def append_paths(server_src_dir, base_stack_folder, service): + """ + Append paths to sys.path in order to import modules. + + @param server_src_dir: Server source directory + @param base_stack_folder: If present, the directory of the stack. + @param service: Service name. + """ + paths_to_add = [] + if base_stack_folder is not None: + # Append paths + metainfo_file = None + if base_stack_folder is not None: + metainfo_file = os.path.join(base_stack_folder, "services", service, "metainfo.xml") + + common_services_parent_dir = os.path.join(server_src_dir, "main", "resources") + paths_to_add = resolve_paths_to_import_from_common_services(metainfo_file, base_stack_folder, common_services_parent_dir, service) + else: + # We couldn't add paths using the base directory, be greedy and add all available for this service in common-services. + # Add the common-services scripts directories to the PATH + base_common_services_folder = os.path.join(server_src_dir, "main", "resources", "common-services") + for folder, subFolders, files in os.walk(os.path.join(base_common_services_folder, service)): + if "package/scripts" in folder: + paths_to_add.append(folder) + + for path in paths_to_add: + if os.path.exists(path) and path not in sys.path: + sys.path.append(path) + def stack_test_executor(base_folder, service, stack, test_mask, executor_result): """ Stack tests executor. Must be executed in separate process to prevent module @@ -95,6 +189,7 @@ def stack_test_executor(base_folder, service, stack, test_mask, executor_result) server_src_dir = get_parent_path(base_folder, 'src') script_folders = set() + base_stack_folder = None if stack is not None: base_stack_folder = os.path.join(server_src_dir, "main", "resources", "stacks", get_stack_name(), stack) @@ -104,16 +199,7 @@ def stack_test_executor(base_folder, service, stack, test_mask, executor_result) if os.path.split(root)[-1] in ["scripts", "files"] and service in root: script_folders.add(root) - # Add the common-services scripts directories to the PATH - base_commserv_folder = os.path.join(server_src_dir, "main", "resources", "common-services") - for folder, subFolders, files in os.walk(os.path.join(base_commserv_folder, service)): - # folder will return the versions of the services - scripts_dir = os.path.join(folder, "package", "scripts") - if os.path.exists(scripts_dir): - script_folders.add(scripts_dir) - - sys.path.extend(script_folders) - + append_paths(server_src_dir, base_stack_folder, service) tests = get_test_files(base_folder, mask = test_mask) #TODO Add an option to randomize the tests' execution