Repository: ambari Updated Branches: refs/heads/branch-2.6 f7d9fd518 -> b531c5f36
AMBARI-22245 - YARN Service Checks Fails Because of Old hadoop-client Classpath Entry (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b531c5f3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b531c5f3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b531c5f3 Branch: refs/heads/branch-2.6 Commit: b531c5f36579aeeb4caa80ea8a9fb8abf2255a1b Parents: f7d9fd5 Author: Jonathan Hurley <[email protected]> Authored: Mon Oct 16 15:33:17 2017 -0400 Committer: Jonathan Hurley <[email protected]> Committed: Mon Oct 16 15:58:35 2017 -0400 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/params_linux.py | 34 +++++++++++-- .../2.1.0.2.0/package/scripts/status_params.py | 52 ++++++++------------ .../services/YARN/configuration/yarn-site.xml | 2 +- .../stacks/HDP/2.6/upgrades/config-upgrade.xml | 7 +++ .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 7 ++- .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 1 + 6 files changed, 63 insertions(+), 40 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b531c5f3/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index f217192..12edc69 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -20,8 +20,10 @@ Ambari Agent """ import os +from resource_management.core import sudo from resource_management.libraries.script.script import Script from resource_management.libraries.resources.hdfs_resource import HdfsResource +from resource_management.libraries.functions import component_version from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import format @@ -88,6 +90,7 @@ stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_D hostname = config['hostname'] # hadoop default parameters +hadoop_home = status_params.hadoop_home hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") hadoop_bin = stack_select.get_hadoop_dir("sbin") hadoop_bin_dir = stack_select.get_hadoop_dir("bin") @@ -113,12 +116,33 @@ if stack_supports_ru: if command_role in YARN_SERVER_ROLE_DIRECTORY_MAP: yarn_role_root = YARN_SERVER_ROLE_DIRECTORY_MAP[command_role] - hadoop_mapred2_jar_location = format("{stack_root}/current/{mapred_role_root}") - mapred_bin = format("{stack_root}/current/{mapred_role_root}/sbin") - + # defaults set to current based on role + hadoop_mapr_home = format("{stack_root}/current/{mapred_role_root}") hadoop_yarn_home = format("{stack_root}/current/{yarn_role_root}") - yarn_bin = format("{stack_root}/current/{yarn_role_root}/sbin") - yarn_container_bin = format("{stack_root}/current/{yarn_role_root}/bin") + + # try to render the specific version + version = component_version.get_component_repository_version() + if version is None: + version = default("/commandParams/version", None) + + + if version is not None: + hadoop_mapr_versioned_home = format("{stack_root}/{version}/hadoop-mapreduce") + hadoop_yarn_versioned_home = format("{stack_root}/{version}/hadoop-yarn") + + if sudo.path_isdir(hadoop_mapr_versioned_home): + hadoop_mapr_home = hadoop_mapr_versioned_home + + if sudo.path_isdir(hadoop_yarn_versioned_home): + hadoop_yarn_home = hadoop_yarn_versioned_home + + + hadoop_mapred2_jar_location = hadoop_mapr_home + mapred_bin = format("{hadoop_mapr_home}/sbin") + + yarn_bin = format("{hadoop_yarn_home}/sbin") + yarn_container_bin = format("{hadoop_yarn_home}/bin") + if stack_supports_timeline_state_store: # Timeline Service property that was added timeline_state_store stack feature http://git-wip-us.apache.org/repos/asf/ambari/blob/b531c5f3/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py index c2e9d92..6bb528f 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/status_params.py @@ -19,43 +19,31 @@ limitations under the License. """ from resource_management.libraries.script.script import Script from resource_management.libraries import functions +from resource_management.libraries.functions import stack_select from resource_management.libraries.functions import format from resource_management.libraries.functions.default import default -from ambari_commons import OSCheck config = Script.get_config() tmp_dir = Script.get_tmp_dir() -if OSCheck.is_windows_family(): - resourcemanager_win_service_name = 'resourcemanager' - nodemanager_win_service_name = 'nodemanager' - historyserver_win_service_name = 'historyserver' - timelineserver_win_service_name = 'timelineserver' - - service_map = { - 'resourcemanager' : resourcemanager_win_service_name, - 'nodemanager' : nodemanager_win_service_name, - 'historyserver' : historyserver_win_service_name, - 'timelineserver' : timelineserver_win_service_name - } -else: - mapred_user = config['configurations']['mapred-env']['mapred_user'] - yarn_user = config['configurations']['yarn-env']['yarn_user'] - yarn_pid_dir_prefix = config['configurations']['yarn-env']['yarn_pid_dir_prefix'] - mapred_pid_dir_prefix = config['configurations']['mapred-env']['mapred_pid_dir_prefix'] - yarn_pid_dir = format("{yarn_pid_dir_prefix}/{yarn_user}") - mapred_pid_dir = format("{mapred_pid_dir_prefix}/{mapred_user}") - - resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanager.pid") - nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid") - yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid") - yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated - mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid") - - hadoop_conf_dir = functions.conf_select.get_hadoop_conf_dir() - - hostname = config['hostname'] - kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) - security_enabled = config['configurations']['cluster-env']['security_enabled'] +mapred_user = config['configurations']['mapred-env']['mapred_user'] +yarn_user = config['configurations']['yarn-env']['yarn_user'] +yarn_pid_dir_prefix = config['configurations']['yarn-env']['yarn_pid_dir_prefix'] +mapred_pid_dir_prefix = config['configurations']['mapred-env']['mapred_pid_dir_prefix'] +yarn_pid_dir = format("{yarn_pid_dir_prefix}/{yarn_user}") +mapred_pid_dir = format("{mapred_pid_dir_prefix}/{mapred_user}") + +resourcemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-resourcemanager.pid") +nodemanager_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-nodemanager.pid") +yarn_historyserver_pid_file_old = format("{yarn_pid_dir}/yarn-{yarn_user}-historyserver.pid") +yarn_historyserver_pid_file = format("{yarn_pid_dir}/yarn-{yarn_user}-timelineserver.pid") # *-historyserver.pid is deprecated +mapred_historyserver_pid_file = format("{mapred_pid_dir}/mapred-{mapred_user}-historyserver.pid") + +hadoop_home = stack_select.get_hadoop_dir("home") +hadoop_conf_dir = functions.conf_select.get_hadoop_conf_dir() + +hostname = config['hostname'] +kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) +security_enabled = config['configurations']['cluster-env']['security_enabled'] stack_name = default("/hostLevelParams/stack_name", None) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b531c5f3/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml index b5eedea..71c63c1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml @@ -19,7 +19,7 @@ <configuration supports_final="true"> <property> <name>yarn.application.classpath</name> - <value>/etc/hadoop/conf,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*,/usr/hdp/current/ext/hadoop/*</value> + <value>{{hadoop_home}}/conf,{{hadoop_home}}/*,{{hadoop_home}}/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*,/usr/hdp/current/ext/hadoop/*</value> <description>Classpath for typical applications.</description> <on-ambari-upgrade add="true"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/b531c5f3/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml index 5dbcf2d..edc8298 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml @@ -161,6 +161,13 @@ if-key="yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor" if-key-state="absent" /> </definition> + + + <definition xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" summary="Application Classpath Parameterization"> + <type>yarn-site</type> + <replace key="yarn.application.classpath" find="/usr/hdp/current/hadoop-client" replace-with="{{hadoop_home}}"/> + </definition> + </changes> </component> <component name="NODEMANAGER"> http://git-wip-us.apache.org/repos/asf/ambari/blob/b531c5f3/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index b7548ea..728bd42 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -278,8 +278,7 @@ <!-- After processing this group, will change the effective Stack of the UpgradeContext object. --> <group xsi:type="update-stack" name="UPDATE_DESIRED_REPOSITORY_ID" title="Update Target Repositories"> <execute-stage title="Update Target Repositories"> - <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.UpdateDesiredRepositoryAction"> - </task> + <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.UpdateDesiredRepositoryAction"/> </execute-stage> </group> @@ -346,6 +345,10 @@ <task xsi:type="configure" id="hdp_2_6_yarn_preemption"/> </execute-stage> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Application Classpath Parameterization"> + <task xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" supports-patch="true"/> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="NODEMANAGER" title="Apply config changes for YARN NM"> <task xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem"> http://git-wip-us.apache.org/repos/asf/ambari/blob/b531c5f3/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index d3a096a..5e9ff7f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -696,6 +696,7 @@ <pre-upgrade> <task xsi:type="configure" id="hdp_2_6_maint_ranger_yarn_plugin_cluster_name"/> <task xsi:type="configure" id="hdp_2_6_yarn_preemption" /> + <task xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" supports-patch="true"/> </pre-upgrade> <pre-downgrade/> <!-- no-op to prevent config changes on downgrade --> <upgrade>
