AMBARI-20469. Add a new property to yarn-site.xml yarn.log.server.web-service.url (Madhuvanthi Radhakrishnan via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8ca09f9f Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8ca09f9f Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8ca09f9f Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 8ca09f9fc80a62d332c100e721b4cc2d2a16ceb3 Parents: 55dc8ac Author: Sumit Mohanty <[email protected]> Authored: Wed Mar 15 17:42:13 2017 -0700 Committer: Sumit Mohanty <[email protected]> Committed: Wed Mar 15 19:27:26 2017 -0700 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 1 + .../services/YARN/configuration/yarn-site.xml | 20 +++++++++ .../stacks/HDP/2.6/services/stack_advisor.py | 44 ++++++++++++++++---- 3 files changed, 58 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8ca09f9f/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 5732a1c..25f2001 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -2487,6 +2487,7 @@ public class BlueprintConfigurationProcessor { yarnSiteMap.put("yarn.timeline-service.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER")); yarnSiteMap.put("yarn.timeline-service.webapp.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER")); yarnSiteMap.put("yarn.timeline-service.webapp.https.address", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER")); + yarnSiteMap.put("yarn.log.server.web-service.url", new SingleHostTopologyUpdater("APP_TIMELINE_SERVER")); // HIVE_SERVER hiveSiteMap.put("hive.server2.authentication.ldap.url", new SingleHostTopologyUpdater("HIVE_SERVER2")); http://git-wip-us.apache.org/repos/asf/ambari/blob/8ca09f9f/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml index 4b97148..0de6f00 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-site.xml @@ -63,4 +63,24 @@ </description> <on-ambari-upgrade add="false"/> </property> + <property> + <name>yarn.log.server.web-service.url</name> + <value>http://localhost:8188/ws/v1/applicationhistory</value> + <description>Log Server Web Service URL.</description> + <depends-on> + <property> + <type>yarn-site</type> + <name>yarn.http.policy</name> + </property> + <property> + <type>yarn-site</type> + <name>yarn.resourcemanager.webapp.address</name> + </property> + <property> + <type>yarn-site</type> + <name>yarn.resourcemanager.webapp.https.address</name> + </property> + </depends-on> + <on-ambari-upgrade add="false"/> + </property> </configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/8ca09f9f/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py index 0968d8b..53ff007 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/stack_advisor.py @@ -167,6 +167,19 @@ class HDP26StackAdvisor(HDP25StackAdvisor): else: ranger_yarn_plugin_enabled = False + #yarn timeline service url depends on http policy and takes the host name of the yarn webapp. + if "yarn-site" in services["configurations"] and \ + "yarn.timeline-service.webapp.https.address" in services["configurations"]["yarn-site"]["properties"] and \ + "yarn.http.policy" in services["configurations"]["yarn-site"]["properties"] and \ + "yarn.log.server.web-service.url" in services["configurations"]["yarn-site"]["properties"]: + if services["configurations"]["yarn-site"]["properties"]["yarn.http.policy"] == 'HTTP_ONLY': + webapp_address = services["configurations"]["yarn-site"]["properties"]["yarn.timeline-service.webapp.address"] + webservice_url = "http://"+webapp_address+"/ws/v1/applicationhistory"; + else: + webapp_address = services["configurations"]["yarn-site"]["properties"]["yarn.timeline-service.webapp.https.address"] + webservice_url = "https://"+webapp_address+"/ws/v1/applicationhistory"; + putYarnSiteProperty('yarn.log.server.web-service.url',webservice_url ) + if ranger_yarn_plugin_enabled and 'ranger-yarn-plugin-properties' in services['configurations'] and 'REPOSITORY_CONFIG_USERNAME' in services['configurations']['ranger-yarn-plugin-properties']['properties']: Logger.info("Setting Yarn Repo user for Ranger.") putRangerYarnPluginProperty = self.putProperty(configurations, "ranger-yarn-plugin-properties", services) @@ -234,7 +247,8 @@ class HDP26StackAdvisor(HDP25StackAdvisor): "DRUID": {"druid-env": self.validateDruidEnvConfigurations, "druid-historical": self.validateDruidHistoricalConfigurations, "druid-broker": self.validateDruidBrokerConfigurations}, - "RANGER": {"ranger-ugsync-site": self.validateRangerUsersyncConfigurations} + "RANGER": {"ranger-ugsync-site": self.validateRangerUsersyncConfigurations}, + "YARN" : {"yarn-site": self.validateYarnSiteConfigurations} } self.mergeValidators(parentValidators, childValidators) return parentValidators @@ -259,6 +273,22 @@ class HDP26StackAdvisor(HDP25StackAdvisor): }) return self.toConfigurationValidationProblems(validationItems, "druid-env") + def validateYarnSiteConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): + validationItems = [] + siteProperties = services["configurations"]["yarn-site"]["properties"] + if services["configurations"]["yarn-site"]["properties"]["yarn.http.policy"] == 'HTTP_ONLY': + webapp_address = services["configurations"]["yarn-site"]["properties"]["yarn.timeline-service.webapp.address"] + propertyValue = "http://"+webapp_address+"/ws/v1/applicationhistory"; + else: + webapp_address = services["configurations"]["yarn-site"]["properties"]["yarn.timeline-service.webapp.https.address"] + propertyValue = "https://"+webapp_address+"/ws/v1/applicationhistory"; + Logger.info("validateYarnSiteConfigurations: recommended value for webservice url"+services["configurations"]["yarn-site"]["properties"]["yarn.log.server.web-service.url"]) + if services["configurations"]["yarn-site"]["properties"]["yarn.log.server.web-service.url"] != propertyValue: + validationItems = [ + {"config-name": "yarn.log.server.web-service.url", + "item": self.getWarnItem("Value should be %s" % propertyValue)}] + return self.toConfigurationValidationProblems(validationItems, "yarn-site") + def validateDruidHistoricalConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): validationItems = [ {"config-name": "druid.processing.numThreads", @@ -268,12 +298,12 @@ class HDP26StackAdvisor(HDP25StackAdvisor): return self.toConfigurationValidationProblems(validationItems, "druid-historical") def validateDruidBrokerConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): - validationItems = [ - {"config-name": "druid.processing.numThreads", - "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, - "druid.processing.numThreads")} - ] - return self.toConfigurationValidationProblems(validationItems, "druid-broker") + validationItems = [ + {"config-name": "druid.processing.numThreads", + "item": self.validatorEqualsToRecommendedItem(properties, recommendedDefaults, + "druid.processing.numThreads")} + ] + return self.toConfigurationValidationProblems(validationItems, "druid-broker") def recommendTezConfigurations(self, configurations, clusterData, services, hosts): super(HDP26StackAdvisor, self).recommendTezConfigurations(configurations, clusterData, services, hosts)
