AMBARI-22678. Fix Broken Symlinks on Stack Distribution (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/72812bd2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/72812bd2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/72812bd2 Branch: refs/heads/branch-feature-AMBARI-20859 Commit: 72812bd2bdd496056407d930c4c7e232bd487703 Parents: dfb9785 Author: Lisnichenko Dmitro <[email protected]> Authored: Thu Jan 4 18:42:11 2018 +0200 Committer: Lisnichenko Dmitro <[email protected]> Committed: Thu Jan 4 18:42:11 2018 +0200 ---------------------------------------------------------------------- .../SPARK/1.2.1/package/scripts/livy_service.py | 2 - .../2.0.0/package/scripts/livy2_service.py | 2 - .../custom_actions/scripts/install_packages.py | 38 ++++--- .../HDP/2.0.6/properties/stack_packages.json | 102 ++++++++++++------- 4 files changed, 92 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/72812bd2/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py index add5a77..0e78881 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/livy_service.py @@ -48,5 +48,3 @@ def livy_service(name, upgrade_type=None, action=None): on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H -E kill -9 {livy_server_pid}"), environment={'JAVA_HOME': params.java_home} ) - - File(params.livy_server_pid_file, action="delete") http://git-wip-us.apache.org/repos/asf/ambari/blob/72812bd2/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py index 0180a31..2ac8fd4 100644 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/livy2_service.py @@ -48,5 +48,3 @@ def livy2_service(name, upgrade_type=None, action=None): on_timeout=format("! ( {process_id_exists_command} ) || {sudo} -H -E kill -9 {livy2_server_pid}"), environment={'JAVA_HOME': params.java_home} ) - - File(params.livy2_server_pid_file, action="delete") http://git-wip-us.apache.org/repos/asf/ambari/blob/72812bd2/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index 4cdb0e5..11b571b 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -154,15 +154,12 @@ class InstallPackages(Script): if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") - self._fix_default_links("livy", "livy-server") - self._fix_default_links("livy2", "livy2-server") - + self._fix_default_links_for_current() # if installing a version of HDP that needs some symlink love, then create them if is_package_install_successful and 'actual_version' in self.structured_output: self._relink_configurations_with_conf_select(stack_id, self.structured_output['actual_version']) - - def _fix_default_links(self, package_name, component_name): + def _fix_default_links_for_current(self): """ If a prior version of Ambari did not correctly reverse the conf symlinks, then they would be put into a bad state when distributing a new stack. For example: @@ -181,19 +178,32 @@ class InstallPackages(Script): :return: None """ + Logger.info("Attempting to fix any configuration symlinks which are not in the correct state") from resource_management.libraries.functions import stack_select - package_dirs = conf_select.get_package_dirs() - if package_name in package_dirs: - Logger.info("Determining if the default conf links for {0} need to be fixed".format(package_name)) + restricted_packages = conf_select.get_restricted_packages() + + if 0 == len(restricted_packages): + Logger.info("There are no restricted conf-select packages for this installation") + else: + Logger.info("Restricting conf-select packages to {0}".format(restricted_packages)) - directories = package_dirs[package_name] - Logger.info("The following directories will be checked for {0}: {1}".format(package_name, - str(directories))) + for package_name, directories in conf_select.get_package_dirs().iteritems(): + Logger.info("Attempting to fix the default conf links for {0}".format(package_name)) + Logger.info("The following directories will be fixed for {0}: {1}".format(package_name, str(directories))) - stack_version = stack_select.get_stack_version_before_install(component_name) - if stack_version: - conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories) + component_name = None + for directory_struct in directories: + if "component" in directory_struct: + component_name = directory_struct["component"] + if component_name: + stack_version = stack_select.get_stack_version_before_install(component_name) + if 0 == len(restricted_packages) or package_name in restricted_packages: + if stack_version: + conf_select.convert_conf_directories_to_symlinks(package_name, stack_version, directories) + else: + Logger.warning( + "Unable to fix {0} since there is no known installed version for this component".format(package_name)) def _relink_configurations_with_conf_select(self, stack_id, stack_version): """ http://git-wip-us.apache.org/repos/asf/ambari/blob/72812bd2/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json index 68da0dd..c3d0d9b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_packages.json @@ -1000,205 +1000,239 @@ "accumulo": [ { "conf_dir": "/etc/accumulo/conf", - "current_dir": "{0}/current/accumulo-client/conf" + "current_dir": "{0}/current/accumulo-client/conf", + "component": "accumulo-client" } ], "atlas": [ { "conf_dir": "/etc/atlas/conf", - "current_dir": "{0}/current/atlas-client/conf" + "current_dir": "{0}/current/atlas-client/conf", + "component": "atlas-client" } ], "druid": [ { "conf_dir": "/etc/druid/conf", - "current_dir": "{0}/current/druid-overlord/conf" + "current_dir": "{0}/current/druid-overlord/conf", + "component": "druid-overlord" } ], "falcon": [ { "conf_dir": "/etc/falcon/conf", - "current_dir": "{0}/current/falcon-client/conf" + "current_dir": "{0}/current/falcon-client/conf", + "component": "falcon-client" } ], "flume": [ { "conf_dir": "/etc/flume/conf", - "current_dir": "{0}/current/flume-server/conf" + "current_dir": "{0}/current/flume-server/conf", + "component": "flume-server" } ], "hadoop": [ { "conf_dir": "/etc/hadoop/conf", - "current_dir": "{0}/current/hadoop-client/conf" + "current_dir": "{0}/current/hadoop-client/conf", + "component": "hadoop-client" } ], "hbase": [ { "conf_dir": "/etc/hbase/conf", - "current_dir": "{0}/current/hbase-client/conf" + "current_dir": "{0}/current/hbase-client/conf", + "component": "hbase-client" } ], "hive": [ { "conf_dir": "/etc/hive/conf", - "current_dir": "{0}/current/hive-client/conf" + "current_dir": "{0}/current/hive-client/conf", + "component": "hive-client" } ], "hive2": [ { "conf_dir": "/etc/hive2/conf", - "current_dir": "{0}/current/hive-server2-hive2/conf" + "current_dir": "{0}/current/hive-server2-hive2/conf", + "component": "hive-server2-hive2" } ], "hive-hcatalog": [ { "conf_dir": "/etc/hive-webhcat/conf", "prefix": "/etc/hive-webhcat", - "current_dir": "{0}/current/hive-webhcat/etc/webhcat" + "current_dir": "{0}/current/hive-webhcat/etc/webhcat", + "component": "hive-webhcat" }, { "conf_dir": "/etc/hive-hcatalog/conf", "prefix": "/etc/hive-hcatalog", - "current_dir": "{0}/current/hive-webhcat/etc/hcatalog" + "current_dir": "{0}/current/hive-webhcat/etc/hcatalog", + "component": "hive-webhcat" } ], "kafka": [ { "conf_dir": "/etc/kafka/conf", - "current_dir": "{0}/current/kafka-broker/conf" + "current_dir": "{0}/current/kafka-broker/conf", + "component": "kafka-broker" } ], "knox": [ { "conf_dir": "/etc/knox/conf", - "current_dir": "{0}/current/knox-server/conf" + "current_dir": "{0}/current/knox-server/conf", + "component": "knox-server" } ], "livy": [ { "conf_dir": "/etc/livy/conf", - "current_dir": "{0}/current/livy-client/conf" + "current_dir": "{0}/current/livy-client/conf", + "component": "livy-client" } ], "livy2": [ { "conf_dir": "/etc/livy2/conf", - "current_dir": "{0}/current/livy2-client/conf" + "current_dir": "{0}/current/livy2-client/conf", + "component": "livy2-client" } ], "mahout": [ { "conf_dir": "/etc/mahout/conf", - "current_dir": "{0}/current/mahout-client/conf" + "current_dir": "{0}/current/mahout-client/conf", + "component": "mahout-client" } ], "nifi": [ { "conf_dir": "/etc/nifi/conf", - "current_dir": "{0}/current/nifi/conf" + "current_dir": "{0}/current/nifi/conf", + "component": "nifi" } ], "oozie": [ { "conf_dir": "/etc/oozie/conf", - "current_dir": "{0}/current/oozie-client/conf" + "current_dir": "{0}/current/oozie-client/conf", + "component": "oozie-client" } ], "phoenix": [ { "conf_dir": "/etc/phoenix/conf", - "current_dir": "{0}/current/phoenix-client/conf" + "current_dir": "{0}/current/phoenix-client/conf", + "component": "phoenix-client" } ], "pig": [ { "conf_dir": "/etc/pig/conf", - "current_dir": "{0}/current/pig-client/conf" + "current_dir": "{0}/current/pig-client/conf", + "component": "pig-client" } ], "ranger-admin": [ { "conf_dir": "/etc/ranger/admin/conf", - "current_dir": "{0}/current/ranger-admin/conf" + "current_dir": "{0}/current/ranger-admin/conf", + "component": "ranger-admin" } ], "ranger-kms": [ { "conf_dir": "/etc/ranger/kms/conf", - "current_dir": "{0}/current/ranger-kms/conf" + "current_dir": "{0}/current/ranger-kms/conf", + "component": "ranger-kms" } ], "ranger-tagsync": [ { "conf_dir": "/etc/ranger/tagsync/conf", - "current_dir": "{0}/current/ranger-tagsync/conf" + "current_dir": "{0}/current/ranger-tagsync/conf", + "component": "ranger-tagsync" } ], "ranger-usersync": [ { "conf_dir": "/etc/ranger/usersync/conf", - "current_dir": "{0}/current/ranger-usersync/conf" + "current_dir": "{0}/current/ranger-usersync/conf", + "component": "ranger-usersync" } ], "slider": [ { "conf_dir": "/etc/slider/conf", - "current_dir": "{0}/current/slider-client/conf" + "current_dir": "{0}/current/slider-client/conf", + "component": "slider-client" } ], "spark": [ { "conf_dir": "/etc/spark/conf", - "current_dir": "{0}/current/spark-client/conf" + "current_dir": "{0}/current/spark-client/conf", + "component": "spark-client" } ], "spark2": [ { "conf_dir": "/etc/spark2/conf", - "current_dir": "{0}/current/spark2-client/conf" + "current_dir": "{0}/current/spark2-client/conf", + "component": "spark2-client" } ], "sqoop": [ { "conf_dir": "/etc/sqoop/conf", - "current_dir": "{0}/current/sqoop-client/conf" + "current_dir": "{0}/current/sqoop-client/conf", + "component": "sqoop-client" } ], "storm": [ { "conf_dir": "/etc/storm/conf", - "current_dir": "{0}/current/storm-client/conf" + "current_dir": "{0}/current/storm-client/conf", + "component": "storm-client" } ], "storm-slider-client": [ { "conf_dir": "/etc/storm-slider-client/conf", - "current_dir": "{0}/current/storm-slider-client/conf" + "current_dir": "{0}/current/storm-slider-client/conf", + "component": "storm-slider-client" } ], "superset": [ { "conf_dir": "/etc/druid-superset/conf", - "current_dir": "{0}/current/druid-superset/conf" + "current_dir": "{0}/current/druid-superset/conf", + "component": "druid-superset" } ], "tez": [ { "conf_dir": "/etc/tez/conf", - "current_dir": "{0}/current/tez-client/conf" + "current_dir": "{0}/current/tez-client/conf", + "component": "tez-client" } ], "zeppelin": [ { "conf_dir": "/etc/zeppelin/conf", - "current_dir": "{0}/current/zeppelin-server/conf" + "current_dir": "{0}/current/zeppelin-server/conf", + "component": "zeppelin-server" } ], "zookeeper": [ { "conf_dir": "/etc/zookeeper/conf", - "current_dir": "{0}/current/zookeeper-client/conf" + "current_dir": "{0}/current/zookeeper-client/conf", + "component": "zookeeper-client" } ] },
