AMBARI-22525 - ATS start failed during patch upgrade due to CNF SparkATSPlugin (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3fa1289e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3fa1289e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3fa1289e Branch: refs/heads/branch-3.0-perf Commit: 3fa1289eaaf9c30985bcf5a50725080a9eb9b368 Parents: 393fdb8 Author: Jonathan Hurley <[email protected]> Authored: Mon Nov 27 13:58:38 2017 -0500 Committer: Jonathan Hurley <[email protected]> Committed: Tue Nov 28 10:24:31 2017 -0500 ---------------------------------------------------------------------- .../libraries/functions/component_version.py | 9 +++++---- .../YARN/2.1.0.2.0/package/scripts/params_linux.py | 6 ++++++ .../YARN/3.0.0.3.0/package/scripts/params_linux.py | 6 ++++++ .../HDP/2.5/services/YARN/configuration/yarn-site.xml | 4 ++-- .../stacks/HDP/2.5/services/stack_advisor.py | 2 +- .../stacks/HDP/2.6/upgrades/config-upgrade.xml | 14 ++++++++++++-- .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 10 ++++++++-- .../resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml | 4 ++++ .../HDP/3.0/services/YARN/configuration/yarn-site.xml | 4 ++-- .../1.0/services/FAKEYARN/configuration/yarn-site.xml | 4 ++-- .../python/stacks/2.5/common/test_stack_advisor.py | 2 +- 11 files changed, 49 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py index 169b339..fd18232 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py @@ -20,7 +20,7 @@ limitations under the License. from resource_management.libraries.script.script import Script -def get_component_repository_version(service_name = None, component_name = None): +def get_component_repository_version(service_name = None, component_name = None, default_value = None): """ Gets the version associated with the specified component from the structure in the command. Every command should contain a mapping of service/component to the desired repository it's set @@ -28,22 +28,23 @@ def get_component_repository_version(service_name = None, component_name = None) :service_name: the name of the service :component_name: the name of the component + :default_value: the value to return if either the service or the component are not found """ config = Script.get_config() versions = _get_component_repositories(config) if versions is None: - return None + return default_value if service_name is None: service_name = config['serviceName'] if config is not None and 'serviceName' in config else None if service_name is None or service_name not in versions: - return None + return default_value component_versions = versions[service_name] if len(component_versions) == 0: - return None + return default_value if component_name is None: component_name = config["role"] if config is not None and "role" in config else None http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 3e4504d..b244af0 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -84,6 +84,12 @@ stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_ # It cannot be used during the initial Cluser Install because the version is not yet known. version = default("/commandParams/version", None) +# these are used to render the classpath for picking up Spark classes +# in the event that spark is not installed, then we must default to the vesrion of YARN installed +# since it will still load classes from its own spark version +spark_version = component_version.get_component_repository_version("SPARK", "SPARK_CLIENT", default_value = version) +spark2_version = component_version.get_component_repository_version("SPARK2", "SPARK2_CLIENT", default_value = version) + stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py index 617dc3b..372ae85 100644 --- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py @@ -84,6 +84,12 @@ stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_ # It cannot be used during the initial Cluser Install because the version is not yet known. version = default("/commandParams/version", None) +# these are used to render the classpath for picking up Spark classes +# in the event that spark is not installed, then we must default to the vesrion of YARN installed +# since it will still load classes from its own spark version +spark_version = component_version.get_component_repository_version("SPARK", "SPARK_CLIENT", default_value = version) +spark2_version = component_version.get_component_repository_version("SPARK2", "SPARK2_CLIENT", default_value = version) + stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT, version_for_stack_feature_checks) stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, version_for_stack_feature_checks) http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml index b6fadcb..29833fb 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml @@ -31,13 +31,13 @@ </property> <property> <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name> - <value>{{stack_root}}/${hdp.version}/spark/aux/*</value> + <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value> <description>The auxiliary service classpath to use for Spark</description> <on-ambari-upgrade add="false"/> </property> <property> <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name> - <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value> + <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value> <description>The auxiliary service classpath to use for Spark 2</description> <on-ambari-upgrade add="false"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index b6f2478..6f409cd 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -823,7 +823,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): if self.isServiceDeployed(services, "SPARK"): timeline_plugin_classes_values.append('org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin') - timeline_plugin_classpath_values.append(stack_root + "/${hdp.version}/spark/hdpLib/*") + timeline_plugin_classpath_values.append(stack_root + "/{{spark_version}}/spark/hdpLib/*") putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes', ",".join(timeline_plugin_classes_values)) putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath', ":".join(timeline_plugin_classpath_values)) http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml index bcb13bc..9311877 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml @@ -162,14 +162,13 @@ if-key-state="absent" /> </definition> - <definition xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" summary="Application Classpath Parameterization"> <type>yarn-site</type> <replace key="yarn.application.classpath" find="/usr/hdp/current/hadoop-client" replace-with="{{hadoop_home}}"/> </definition> - </changes> </component> + <component name="NODEMANAGER"> <changes> <definition xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem"> @@ -179,6 +178,17 @@ </definition> </changes> </component> + + <component name="APP_TIMELINE_SERVER"> + <changes> + <definition xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths"> + <type>yarn-site</type> + <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/{{spark_version}}/spark/aux/*"/> + <set key="yarn.nodemanager.aux-services.spark2_shuffle.classpath" value="{{stack_root}}/{{spark2_version}}/spark2/aux/*"/> + <replace key="yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath" find="${hdp.version}/spark/" replace-with="{{spark_version}}/spark/"/> + </definition> + </changes> + </component> </service> <service name="KAFKA"> http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml index 0355362..2184060 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml @@ -147,6 +147,7 @@ <component>DRUID_SUPERSET</component> </service> + </group> <group xsi:type="cluster" name="Backups" title="Perform Backups"> @@ -351,7 +352,12 @@ <task xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" supports-patch="true"/> </execute-stage> - <!-- YARN --> + <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Spark Versioned Classpaths"> + <task xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths" supports-patch="true"> + <summary>Updating the Spark classpath entries to use the specific version of Spark which is installed.</summary> + </task> + </execute-stage> + <execute-stage service="YARN" component="NODEMANAGER" title="Apply config changes for YARN NM"> <task xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem"> <summary>Updating YARN NodeManager config for LLAP</summary> @@ -414,7 +420,7 @@ <execute-stage service="TEZ" component="TEZ_CLIENT" title="Updating LD Library Classpath for Tarball" > <task xsi:type="configure" id="hdp_2_6_tez_tarball_ld_library" supports-patch="true"/> </execute-stage> - + <!-- MapR --> <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Updating LD Library Classpath for Tarball"> <task xsi:type="configure" id="hdp_2_6_mapreduce_tarball_ld_library" supports-patch="true"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml index 5aa2d20..70f84ab 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml @@ -690,6 +690,10 @@ <service name="YARN"> <component name="APP_TIMELINE_SERVER"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths" supports-patch="true"/> + </pre-upgrade> + <pre-downgrade/> <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml index 01551c4..d32e2de 100644 --- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml @@ -20,7 +20,7 @@ <property> <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name> - <value>{{stack_root}}/${hdp.version}/spark/aux/*</value> + <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value> <description>The auxiliary service classpath to use for Spark</description> <on-ambari-upgrade add="false"/> </property> @@ -41,7 +41,7 @@ <!-- These configs were inherited from HDP 2.5 --> <property> <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name> - <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value> + <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value> <description>The auxiliary service classpath to use for Spark 2</description> <on-ambari-upgrade add="false"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml index 57d03a8..392d0de 100644 --- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml @@ -747,13 +747,13 @@ </property> <property> <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name> - <value>{{stack_root}}/${hdp.version}/spark/aux/*</value> + <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value> <description>The auxiliary service classpath to use for Spark</description> <on-ambari-upgrade add="false"/> </property> <property> <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name> - <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value> + <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value> <description>The auxiliary service classpath to use for Spark 2</description> <on-ambari-upgrade add="false"/> </property> http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 407e78d..cc6bd9d 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -1206,7 +1206,7 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], 'default') self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes'], 'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl,org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin') - self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/${hdp.version}/spark/hdpLib/*') + self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/{{spark_version}}/spark/hdpLib/*') self.assertTrue('hive-interactive-env' not in configurations) self.assertTrue('property_attributes' not in configurations)
