Repository: ambari Updated Branches: refs/heads/branch-3.0-perf 6578b5a28 -> cfec01d69
AMBARI-21519. Fix custom actions to handle changed config paths (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cfec01d6 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cfec01d6 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cfec01d6 Branch: refs/heads/branch-3.0-perf Commit: cfec01d69ffedfef23890c09b540e596a229bded Parents: 6578b5a Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Tue Aug 8 12:48:32 2017 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Tue Aug 8 12:48:32 2017 +0300 ---------------------------------------------------------------------- .../custom_actions/scripts/check_host.py | 56 ++++++++++---------- .../custom_actions/scripts/install_packages.py | 8 +-- .../custom_actions/scripts/ru_execute_tasks.py | 2 +- .../custom_actions/scripts/ru_set_all.py | 2 +- .../custom_actions/scripts/update_repo.py | 2 +- 5 files changed, 36 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/cfec01d6/ambari-server/src/main/resources/custom_actions/scripts/check_host.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py index 358c104..53fad7e 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/check_host.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/check_host.py @@ -86,13 +86,13 @@ class CheckHost(Script): "^slider.*$", "^sqoop.*$", "^storm.*$", "^flume.*$","^hcatalog.*$", "^phoenix.*$", "^ranger.*$", "^accumulo.*$", "^hive_.*$", "^pig[_\-.].*$" # there's a default 'pigz' package which we should avoid ] - + # ignore packages from repos whose names start with these strings IGNORE_PACKAGES_FROM_REPOS = [ "installed" ] - + # ignore required packages IGNORE_PACKAGES = [ @@ -100,20 +100,20 @@ class CheckHost(Script): # ganglia related: "ganglia", "libganglia", "libconfuse", "perl", "rrdtool", "python-rrdtool", "gmetad", "librrd", "rrdcached" ] - + # Additional packages to look for (search packages that start with these) ADDITIONAL_PACKAGES = [ "ambari-log4j" ] - + # ignore repos from the list of repos to be cleaned IGNORE_REPOS = [ "HDP-UTILS", "AMBARI", "BASE", "EXTRAS" ] - + def __init__(self): self.reportFileHandler = HostCheckReportFileHandler() - + def actionexecute(self, env): Logger.info("Host checks started.") config = Script.get_config() @@ -129,6 +129,8 @@ class CheckHost(Script): Logger.info("Check execute list: " + str(check_execute_list)) + raise Exception("SKipping") + # check each of the commands; if an unknown exception wasn't handled # by the functions, then produce a generic exit_code : 1 if CHECK_JAVA_HOME in check_execute_list: @@ -161,7 +163,7 @@ class CheckHost(Script): except Exception, exception : Logger.exception("There was an unknown error while checking last host environment details: " + str(exception)) structured_output[CHECK_LAST_AGENT_ENV] = {"exit_code" : 1, "message": str(exception)} - + # CHECK_INSTALLED_PACKAGES and CHECK_EXISTING_REPOS required to run together for # reasons of not doing the same common work twice for them as it takes some time, especially on Ubuntu. if CHECK_INSTALLED_PACKAGES in check_execute_list and CHECK_EXISTING_REPOS in check_execute_list: @@ -185,7 +187,7 @@ class CheckHost(Script): # this is necessary for HostCleanup to know later what were the results. self.reportFileHandler.writeHostChecksCustomActionsFile(structured_output) - + self.put_structured_out(structured_output) error_message = "" @@ -241,7 +243,7 @@ class CheckHost(Script): additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames( self.ADDITIONAL_PACKAGES, installedPackages) allPackages = list(set(packagesInstalled + additionalPkgsInstalled)) - + installedPackages = packages_analyzer.getPackageDetails(installedPackages, allPackages) repos = packages_analyzer.getReposToRemove(repos, self.IGNORE_REPOS) @@ -257,7 +259,7 @@ class CheckHost(Script): java_bin = "java" if OSCheck.is_windows_family(): java_bin = "java.exe" - + if not os.path.isfile(os.path.join(java_home, "bin", java_bin)): Logger.warning("Java home doesn't exist!") java_home_check_structured_output = {"exit_code" : 1, "message": "Java home doesn't exist!"} @@ -271,9 +273,9 @@ class CheckHost(Script): def execute_db_connection_check(self, config, tmp_dir): Logger.info("DB connection check started.") - + # initialize needed data - + ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] @@ -330,7 +332,7 @@ class CheckHost(Script): db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] - agent_cache_dir = os.path.abspath(config["hostLevelParams"]["agentCacheDir"]) + agent_cache_dir = os.path.abspath(config["agentLevelParams"]["agentCacheDir"]) check_db_connection_url = jdk_location + check_db_connection_jar_name jdbc_path = os.path.join(agent_cache_dir, jdbc_name) class_path_delimiter = ":" @@ -421,7 +423,7 @@ class CheckHost(Script): Logger.exception(message) db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output - + # download jdbc driver from ambari-server resources try: download_file(jdbc_url, jdbc_path) @@ -468,24 +470,24 @@ class CheckHost(Script): # check whether each host in the command can be resolved to an IP address def execute_host_resolution_check(self, config): Logger.info("IP address forward resolution check started.") - + FORWARD_LOOKUP_REASON = "FORWARD_LOOKUP" - + failedCount = 0 failures = [] hosts_with_failures = [] - + if config['commandParams']['hosts'] is not None : hosts = config['commandParams']['hosts'].split(",") successCount = len(hosts) else : successCount = 0 hosts = "" - - socket.setdefaulttimeout(3) + + socket.setdefaulttimeout(3) for host in hosts: try: - host = host.strip() + host = host.strip() socket.gethostbyname(host) except socket.error,exception: successCount -= 1 @@ -493,22 +495,22 @@ class CheckHost(Script): hosts_with_failures.append(host) - failure = { "host": host, "type": FORWARD_LOOKUP_REASON, + failure = { "host": host, "type": FORWARD_LOOKUP_REASON, "cause": exception.args } - + failures.append(failure) - + if failedCount > 0 : message = "There were " + str(failedCount) + " host(s) that could not resolve to an IP address." else : message = "All hosts resolved to an IP address." Logger.info(message) - + host_resolution_check_structured_output = { "exit_code" : 0, - "message" : message, - "failed_count" : failedCount, + "message" : message, + "failed_count" : failedCount, "success_count" : successCount, "failures" : failures, "hosts_with_failures" : hosts_with_failures @@ -522,7 +524,7 @@ class CheckHost(Script): Logger.info("Last Agent Env check started.") hostInfo = HostInfo() last_agent_env_check_structured_output = { } - hostInfo.register(last_agent_env_check_structured_output, False, False) + hostInfo.register(last_agent_env_check_structured_output, runExpensiveChecks=False, checkJavaProcs=True) Logger.info("Last Agent Env check completed successfully.") return last_agent_env_check_structured_output http://git-wip-us.apache.org/repos/asf/ambari/blob/cfec01d6/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index 33adce1..6509e2b 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -92,8 +92,8 @@ class InstallPackages(Script): # current stack information self.current_stack_version_formatted = None - if 'stack_version' in config['hostLevelParams']: - current_stack_version_unformatted = str(config['hostLevelParams']['stack_version']) + if 'stack_version' in config['clusterLevelParams']: + current_stack_version_unformatted = str(config['clusterLevelParams']['stack_version']) self.current_stack_version_formatted = format_stack_version(current_stack_version_unformatted) @@ -350,8 +350,8 @@ class InstallPackages(Script): ret_code = 0 config = self.get_config() - agent_stack_retry_on_unavailability = cbool(config['hostLevelParams']['agent_stack_retry_on_unavailability']) - agent_stack_retry_count = cint(config['hostLevelParams']['agent_stack_retry_count']) + agent_stack_retry_on_unavailability = cbool(config['ambariLevelParams']['agent_stack_retry_on_unavailability']) + agent_stack_retry_count = cint(config['ambariLevelParams']['agent_stack_retry_count']) # Install packages packages_were_checked = False http://git-wip-us.apache.org/repos/asf/ambari/blob/cfec01d6/ambari-server/src/main/resources/custom_actions/scripts/ru_execute_tasks.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/ru_execute_tasks.py b/ambari-server/src/main/resources/custom_actions/scripts/ru_execute_tasks.py index 4dec16f..b4dfb30 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/ru_execute_tasks.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/ru_execute_tasks.py @@ -113,7 +113,7 @@ class ExecuteUpgradeTasks(Script): if task.script and task.function: file_cache = FileCache(agent_config) - server_url_prefix = default('/hostLevelParams/jdk_location', "") + server_url_prefix = default('/ambariLevelParams/jdk_location', "") if service_package_folder and hooks_folder: command_paths = { http://git-wip-us.apache.org/repos/asf/ambari/blob/cfec01d6/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py index a7732d9..ab75be0 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/ru_set_all.py @@ -84,7 +84,7 @@ class UpgradeSetAll(Script): Reverses the work performed in link_config. This should only be used when downgrading from HDP 2.3 to 2.2 in order to under the symlink work required for 2.3. """ - stack_name = default('/hostLevelParams/stack_name', "").upper() + stack_name = default('/clusterLevelParams/stack_name', "").upper() downgrade_to_version = default('/commandParams/version', None) downgrade_from_version = default('/commandParams/downgrade_from_version', None) upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE) http://git-wip-us.apache.org/repos/asf/ambari/blob/cfec01d6/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py b/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py index 9f2107d..42593cc 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/update_repo.py @@ -37,7 +37,7 @@ class UpdateRepo(Script): try: - repo_info_json = config['hostLevelParams']['repo_info'] + repo_info_json = config['hostLevelParams']['repoInfo'] repo_info_dict = json.loads(repo_info_json) for item in repo_info_dict["repositories"]:
