Repository: ambari Updated Branches: refs/heads/branch-2.6 4fe795dba -> 2cf8cb85b
AMBARI-22525 - ATS start failed during patch upgrade due to CNF SparkATSPlugin (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2cf8cb85 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2cf8cb85 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2cf8cb85 Branch: refs/heads/branch-2.6 Commit: 2cf8cb85bffff266897e5941deaad2e92ae53447 Parents: 4fe795d Author: Jonathan Hurley <[email protected]> Authored: Mon Nov 27 13:58:38 2017 -0500 Committer: Jonathan Hurley <[email protected]> Committed: Tue Nov 28 09:24:50 2017 -0500 ---------------------------------------------------------------------- .../libraries/functions/component_version.py | 9 +++++---- .../YARN/2.1.0.2.0/package/scripts/params_linux.py | 6 ++++++ .../BigInsights/4.0/stack-advisor/stack_advisor_25.py | 2 +- .../HDP/2.5/services/YARN/configuration/yarn-site.xml | 4 ++-- .../stacks/HDP/2.5/services/stack_advisor.py | 2 +- .../stacks/HDP/2.6/upgrades/config-upgrade.xml | 14 ++++++++++++-- .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 11 ++++++++--- .../resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 4 ++++ .../python/stacks/2.5/common/test_stack_advisor.py | 2 +- 9 files changed, 40 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py index 169b339..fd18232 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py @@ -20,7 +20,7 @@ limitations under the License. from resource_management.libraries.script.script import Script -def get_component_repository_version(service_name = None, component_name = None): +def get_component_repository_version(service_name = None, component_name = None, default_value = None): """ Gets the version associated with the specified component from the structure in the command. Every command should contain a mapping of service/component to the desired repository it's set @@ -28,22 +28,23 @@ def get_component_repository_version(service_name = None, component_name = None) :service_name: the name of the service :component_name: the name of the component + :default_value: the value to return if either the service or the component are not found """ config = Script.get_config() versions = _get_component_repositories(config) if versions is None: - return None + return default_value if service_name is None: service_name = config['serviceName'] if config is not None and 'serviceName' in config else None if service_name is None or service_name not in versions: - return None + return default_value component_versions = versions[service_name] if len(component_versions) == 0: - return None + return default_value if component_name is None: component_name = config["role"] if config is not None and "role" in config else None http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 4079f18..2871dc0 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -84,6 +84,12 @@ stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_ # It cannot be used during the initial Cluser Install because the version is not yet known. version = default("/commandParams/version", None) +# these are used to render the classpath for picking up Spark classes +# in the event that spark is not installed, then we must default to the vesrion of YARN installed +# since it will still load classes from its own spark version +spark_version = component_version.get_component_repository_version("SPARK", "SPARK_CLIENT", default_value = version) +spark2_version = component_version.get_component_repository_version("SPARK2", "SPARK2_CLIENT", default_value = version) + stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py index 7e77382..ead9e83 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py @@ -745,7 +745,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): if self.__isServiceDeployed(services, "SPARK"): timeline_plugin_classes_values.append('org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin') - timeline_plugin_classpath_values.append(stack_root + "/${hdp.version}/spark/hdpLib/*") + timeline_plugin_classpath_values.append(stack_root + "/{{spark_version}}/spark/hdpLib/*") putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', ",".join(timeline_plugin_classes_values)) putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath', ":".join(timeline_plugin_classpath_values)) http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml index b6fadcb..29833fb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml @@ -31,13 +31,13 @@ </property> <property> <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name> - <value>{{stack_root}}/${hdp.version}/spark/aux/*</value> + <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value> <description>The auxiliary service classpath to use for Spark</description> <on-ambari-upgrade add="false"/> </property> <property> <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name> - <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value> + <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value> <description>The auxiliary service classpath to use for Spark 2</description> <on-ambari-upgrade add="false"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index a3b8263..4187dc9 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -824,7 +824,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): if self.__isServiceDeployed(services, "SPARK"): timeline_plugin_classes_values.append('org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin') - timeline_plugin_classpath_values.append(stack_root + "/${hdp.version}/spark/hdpLib/*") + timeline_plugin_classpath_values.append(stack_root + "/{{spark_version}}/spark/hdpLib/*") putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', ",".join(timeline_plugin_classes_values)) putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath', ":".join(timeline_plugin_classpath_values)) http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml index 4538072..42dc124 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml @@ -168,14 +168,13 @@ if-key-state="absent" /> </definition> - <definition xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" summary="Application Classpath Parameterization"> <type>yarn-site</type> <replace key="yarn.application.classpath" find="/usr/hdp/current/hadoop-client" replace-with="{{hadoop_home}}"/> </definition> - </changes> </component> + <component name="NODEMANAGER"> <changes> <definition xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem"> @@ -185,6 +184,17 @@ </definition> </changes> </component> + + <component name="APP_TIMELINE_SERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths"> + <type>yarn-site</type> + <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/{{spark_version}}/spark/aux/*"/> + <set key="yarn.nodemanager.aux-services.spark2_shuffle.classpath" value="{{stack_root}}/{{spark2_version}}/spark2/aux/*"/> + <replace key="yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath" find="${hdp.version}/spark/" replace-with="{{spark_version}}/spark/"/> + </definition> + </changes> + </component> </service> <service name="KAFKA"> http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index f3cea48..2185049 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -146,7 +146,7 @@ <component>DRUID_OVERLORD</component> <component>DRUID_SUPERSET</component> </service> - + <service name="RANGER"> <component>RANGER_USERSYNC</component> <component>RANGER_ADMIN</component> @@ -354,7 +354,12 @@ <task xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" supports-patch="true"/> </execute-stage> - <!-- YARN --> + <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Spark Versioned Classpaths"> + <task xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths" supports-patch="true"> + <summary>Updating the Spark classpath entries to use the specific version of Spark which is installed.</summary> + </task> + </execute-stage> + <execute-stage service="YARN" component="NODEMANAGER" title="Apply config changes for YARN NM"> <task xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem"> <summary>Updating YARN NodeManager config for LLAP</summary> @@ -417,7 +422,7 @@ <execute-stage service="TEZ" component="TEZ_CLIENT" title="Updating LD Library Classpath for Tarball" > <task xsi:type="configure" id="hdp_2_6_tez_tarball_ld_library" supports-patch="true"/> </execute-stage> - + <!-- MapR --> <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Updating LD Library Classpath for Tarball"> <task xsi:type="configure" id="hdp_2_6_mapreduce_tarball_ld_library" supports-patch="true"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index 3bb6322..d51706a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -690,6 +690,10 @@ <service name="YARN"> <component name="APP_TIMELINE_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths" supports-patch="true"/> + </pre-upgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/2cf8cb85/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index bb11969..8f27d79 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -1206,7 +1206,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], 'default') self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes'], 'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl,org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin') - self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/${hdp.version}/spark/hdpLib/*') + self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/{{spark_version}}/spark/hdpLib/*') self.assertTrue('hive-interactive-env' not in configurations) self.assertTrue('property_attributes' not in configurations)
