AMBARI-21471. ATS going down due to missing org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin (smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/afea7bb7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/afea7bb7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/afea7bb7 Branch: refs/heads/branch-feature-AMBARI-21348 Commit: afea7bb72475fa3d6938aa049dd2db99f3adc133 Parents: d2c6d53 Author: Sumit Mohanty <[email protected]> Authored: Thu Jul 13 22:35:28 2017 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Thu Jul 13 22:35:28 2017 -0700 ---------------------------------------------------------------------- .../4.0/stack-advisor/stack_advisor_25.py | 4 +- .../stacks/HDP/2.5/services/stack_advisor.py | 5 +- .../src/main/resources/stacks/stack_advisor.py | 18 +++ .../stacks/2.5/common/test_stack_advisor.py | 150 +++++++++++-------- 4 files changed, 106 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/afea7bb7/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py index 1f0ae18..eb7d370 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/stack-advisor/stack_advisor_25.py @@ -734,9 +734,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME) putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services) - stack_root = "/usr/hdp" - if "cluster-env" in services["configurations"] and "stack_root" in services["configurations"]["cluster-env"]["properties"]: - stack_root = services["configurations"]["cluster-env"]["properties"]["stack_root"] + stack_root = self.getStackRoot(services) timeline_plugin_classes_values = [] timeline_plugin_classpath_values = [] http://git-wip-us.apache.org/repos/asf/ambari/blob/afea7bb7/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py index 638e79a..97c49f3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py @@ -19,6 +19,7 @@ limitations under the License. import math + from ambari_commons.str_utils import string_set_equals from resource_management.core.logger import Logger from resource_management.core.exceptions import Fail @@ -775,9 +776,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor): self.checkAndStopLlapQueue(services, configurations, LLAP_QUEUE_NAME) putYarnSiteProperty = self.putProperty(configurations, "yarn-site", services) - stack_root = "/usr/hdp" - if cluster_env and "stack_root" in cluster_env: - stack_root = cluster_env["stack_root"] + stack_root = self.getStackRoot(services) timeline_plugin_classes_values = [] timeline_plugin_classpath_values = [] http://git-wip-us.apache.org/repos/asf/ambari/blob/afea7bb7/ambari-server/src/main/resources/stacks/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py index e393b9c..1e0d83a 100644 --- a/ambari-server/src/main/resources/stacks/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/stack_advisor.py @@ -22,6 +22,7 @@ import os import re import socket import traceback +import json class StackAdvisor(object): """ @@ -1144,6 +1145,23 @@ class DefaultStackAdvisor(StackAdvisor): return mount_points + def getStackRoot(self, services): + """ + Gets the stack root associated with the stack + :param services: the services structure containing the current configurations + :return: the stack root as specified in the config or /usr/hdp + """ + cluster_env = self.getServicesSiteProperties(services, "cluster-env") + stack_root = "/usr/hdp" + if cluster_env and "stack_root" in cluster_env: + stack_root_as_str = cluster_env["stack_root"] + stack_roots = json.loads(stack_root_as_str) + stack_name = cluster_env["stack_name"] + if stack_name in stack_roots: + stack_root = stack_roots[stack_name] + + return stack_root + def isSecurityEnabled(self, services): """ Determines if security is enabled by testing the value of cluster-env/security enabled. http://git-wip-us.apache.org/repos/asf/ambari/blob/afea7bb7/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py index 7bc9272..222f100 100644 --- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py +++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py @@ -825,70 +825,80 @@ class TestHDP25StackAdvisor(TestCase): services = { "services": [{ - "StackServices": { - "service_name": "YARN", - }, - "Versions": { - "stack_version": "2.5" - }, - "components": [ - { - "StackServiceComponents": { - "component_name": "NODEMANAGER", - "hostnames": ["c6401.ambari.apache.org"] - } - } - ] - }, { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", - "StackServices": { - "service_name": "HIVE", - "service_version": "1.2.1.2.5", - "stack_name": "HDP", - "stack_version": "2.5" - }, - "components": [ - { - "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", - "StackServiceComponents": { - "advertise_version": "true", - "bulk_commands_display_name": "", - "bulk_commands_master_component_name": "", - "cardinality": "0-1", - "component_category": "MASTER", - "component_name": "HIVE_SERVER_INTERACTIVE", - "custom_commands": ["RESTART_LLAP"], - "decommission_allowed": "false", - "display_name": "HiveServer2 Interactive", - "has_bulk_commands_definition": "false", - "is_client": "false", - "is_master": "true", - "reassign_allowed": "false", - "recovery_enabled": "false", - "service_name": "HIVE", - "stack_name": "HDP", - "stack_version": "2.5", - "hostnames": ["c6401.ambari.apache.org"] - }, - "dependencies": [] - }, - { - "StackServiceComponents": { - "advertise_version": "true", - "cardinality": "1+", - "component_category": "SLAVE", - "component_name": "NODEMANAGER", - "display_name": "NodeManager", - "is_client": "false", - "is_master": "false", - "hostnames": [ - "c6403.ambari.apache.org" - ] - }, - "dependencies": [] - }, - ] - } + "StackServices": { + "service_name": "TEZ" + } + }, + { + "StackServices": { + "service_name": "SPARK" + } + }, + { + "StackServices": { + "service_name": "YARN", + }, + "Versions": { + "stack_version": "2.5" + }, + "components": [ + { + "StackServiceComponents": { + "component_name": "NODEMANAGER", + "hostnames": ["c6401.ambari.apache.org"] + } + } + ] + }, { + "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE", + "StackServices": { + "service_name": "HIVE", + "service_version": "1.2.1.2.5", + "stack_name": "HDP", + "stack_version": "2.5" + }, + "components": [ + { + "href": "/api/v1/stacks/HDP/versions/2.5/services/HIVE/components/HIVE_SERVER_INTERACTIVE", + "StackServiceComponents": { + "advertise_version": "true", + "bulk_commands_display_name": "", + "bulk_commands_master_component_name": "", + "cardinality": "0-1", + "component_category": "MASTER", + "component_name": "HIVE_SERVER_INTERACTIVE", + "custom_commands": ["RESTART_LLAP"], + "decommission_allowed": "false", + "display_name": "HiveServer2 Interactive", + "has_bulk_commands_definition": "false", + "is_client": "false", + "is_master": "true", + "reassign_allowed": "false", + "recovery_enabled": "false", + "service_name": "HIVE", + "stack_name": "HDP", + "stack_version": "2.5", + "hostnames": ["c6401.ambari.apache.org"] + }, + "dependencies": [] + }, + { + "StackServiceComponents": { + "advertise_version": "true", + "cardinality": "1+", + "component_category": "SLAVE", + "component_name": "NODEMANAGER", + "display_name": "NodeManager", + "is_client": "false", + "is_master": "false", + "hostnames": [ + "c6403.ambari.apache.org" + ] + }, + "dependencies": [] + }, + ] + } ], "changed-configurations": [ { @@ -898,6 +908,12 @@ class TestHDP25StackAdvisor(TestCase): } ], "configurations": { + "cluster-env": { + "properties": { + "stack_root": "{\"HDP\":\"/usr/hdp\"}", + "stack_name": "HDP" + }, + }, "capacity-scheduler": { "properties": { "capacity-scheduler": 'yarn.scheduler.capacity.root.default.maximum-capacity=60\n' @@ -960,7 +976,8 @@ class TestHDP25StackAdvisor(TestCase): "tez.am.resource.memory.mb": "341" } } - } + }, + "ambari-server-properties": {"ambari-server.user":"ambari_user"} } clusterData = { @@ -990,6 +1007,9 @@ class TestHDP25StackAdvisor(TestCase): self.assertEquals(configurations['hive-interactive-site']['properties']['hive.server2.tez.default.queues'], 'default') self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'], 'default') + self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes'], + 'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl,org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin') + self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'], '/usr/hdp/${hdp.version}/spark/hdpLib/*') self.assertTrue('hive-interactive-env' not in configurations) self.assertTrue('property_attributes' not in configurations)
