Repository: ambari Updated Branches: refs/heads/branch-2.4 752dc9d8a -> 8b89e478d
AMBARI-17255 AMS collector not coming up in 2.5 cluster blueprint deployment (dsen) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8b89e478 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8b89e478 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8b89e478 Branch: refs/heads/branch-2.4 Commit: 8b89e478dbaa3c4f283f97d7605495dbdf2b14ab Parents: 752dc9d Author: Dmytro Sen <d...@apache.org> Authored: Fri Jun 17 11:27:16 2016 +0300 Committer: Dmytro Sen <d...@apache.org> Committed: Fri Jun 17 11:32:17 2016 +0300 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 13 ++-- .../package/alerts/alert_metrics_deviation.py | 25 ++++--- .../BlueprintConfigurationProcessorTest.java | 2 +- .../2.0.6/HDFS/test_alert_metrics_deviation.py | 70 ++++++++++++++------ 4 files changed, 74 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8b89e478/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 1e56bef..c3af4eb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -2716,15 +2716,18 @@ public class BlueprintConfigurationProcessor { amsSiteMap.put("timeline.metrics.service.webapp.address", new SingleHostTopologyUpdater("METRICS_COLLECTOR") { @Override public String updateForClusterCreate(String propertyName, String origValue, Map<String, Map<String, String>> properties, ClusterTopology topology) { - String value = origValue; - if (isSpecialNetworkAddress(origValue)) { - value = origValue.replace(BIND_ALL_IP_ADDRESS, "localhost"); - } int metricsCollectorsCount = topology.getHostAssignmentsForComponent("METRICS_COLLECTOR").size(); if (metricsCollectorsCount == 1) { + String value = origValue; + //localhost will be replaced with real hostname in updateForClusterCreate() + if (isSpecialNetworkAddress(origValue)) { + value = origValue.replace(BIND_ALL_IP_ADDRESS, "localhost"); + } return super.updateForClusterCreate(propertyName, value, properties, topology); + } else { + //For multiple collectors + return origValue.replace("localhost", BIND_ALL_IP_ADDRESS); } - return origValue; } }); } http://git-wip-us.apache.org/repos/asf/ambari/blob/8b89e478/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py index 9ff1273..f0a1d5c 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/alerts/alert_metrics_deviation.py @@ -55,6 +55,8 @@ SMOKEUSER_KEY = '{{cluster-env/smokeuser}}' EXECUTABLE_SEARCH_PATHS = '{{kerberos-env/executable_search_paths}}' METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY = '{{ams-site/timeline.metrics.service.webapp.address}}' +METRICS_COLLECTOR_VIP_HOST_KEY = '{{cluster-env/metrics_collector_vip_host}}' +METRICS_COLLECTOR_VIP_PORT_KEY = '{{cluster-env/metrics_collector_vip_port}}' CONNECTION_TIMEOUT_KEY = 'connection.timeout' CONNECTION_TIMEOUT_DEFAULT = 5.0 @@ -101,6 +103,7 @@ def get_tokens(): return (HDFS_SITE_KEY, NAMESERVICE_KEY, NN_HTTP_ADDRESS_KEY, DFS_POLICY_KEY, EXECUTABLE_SEARCH_PATHS, NN_HTTPS_ADDRESS_KEY, SMOKEUSER_KEY, KERBEROS_KEYTAB, KERBEROS_PRINCIPAL, SECURITY_ENABLED_KEY, + METRICS_COLLECTOR_VIP_HOST_KEY, METRICS_COLLECTOR_VIP_PORT_KEY, METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY) def execute(configurations={}, parameters={}, host_name=None): @@ -164,17 +167,21 @@ def execute(configurations={}, parameters={}, host_name=None): if not HDFS_SITE_KEY in configurations: return (RESULT_STATE_UNKNOWN, ['{0} is a required parameter for the script'.format(HDFS_SITE_KEY)]) - # ams-site/timeline.metrics.service.webapp.address is required - if not METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY in configurations: - return (RESULT_STATE_UNKNOWN, ['{0} is a required parameter for the script'.format(METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY)]) + if METRICS_COLLECTOR_VIP_HOST_KEY in configurations and METRICS_COLLECTOR_VIP_PORT_KEY in configurations: + collector_host = configurations[METRICS_COLLECTOR_VIP_HOST_KEY] + collector_port = int(configurations[METRICS_COLLECTOR_VIP_PORT_KEY]) else: - collector_webapp_address = configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY].split(":") - if valid_collector_webapp_address(collector_webapp_address): - collector_host = collector_webapp_address[0] - collector_port = int(collector_webapp_address[1]) + # ams-site/timeline.metrics.service.webapp.address is required + if not METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY in configurations: + return (RESULT_STATE_UNKNOWN, ['{0} is a required parameter for the script'.format(METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY)]) else: - return (RESULT_STATE_UNKNOWN, ['{0} value should be set as "fqdn_hostname:port", but set to {1}'.format( - METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY])]) + collector_webapp_address = configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY].split(":") + if valid_collector_webapp_address(collector_webapp_address): + collector_host = collector_webapp_address[0] + collector_port = int(collector_webapp_address[1]) + else: + return (RESULT_STATE_UNKNOWN, ['{0} value should be set as "fqdn_hostname:port", but set to {1}'.format( + METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY, configurations[METRICS_COLLECTOR_WEBAPP_ADDRESS_KEY])]) namenode_service_rpc_address = None # hdfs-site is required http://git-wip-us.apache.org/repos/asf/ambari/blob/8b89e478/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java index a927548..b595868 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessorTest.java @@ -7808,7 +7808,7 @@ public class BlueprintConfigurationProcessorTest { Map<String, String> amsSite = new HashMap<String, String>(); //default - amsSite.put("timeline.metrics.service.webapp.address", "0.0.0.0:6188"); + amsSite.put("timeline.metrics.service.webapp.address", "localhost:6188"); properties.put("ams-site", amsSite); Map<String, Map<String, String>> parentProperties = new HashMap<String, Map<String, String>>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/8b89e478/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py index b7d96b7..5bfbfc6 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_alert_metrics_deviation.py @@ -51,27 +51,6 @@ class TestAlertMetricsDeviation(RMFTestCase): sys.path.append(file_path) global alert import alert_metrics_deviation as alert - global configs - configs = { - '{{hdfs-site/dfs.namenode.https-address}}': 'c6401.ambari.apache.org:50470', - '{{hdfs-site/dfs.http.policy}}': 'HTTP_ONLY', - '{{ams-site/timeline.metrics.service.webapp.address}}': 'c6401.ambari.apache.org:6188', - '{{hdfs-site/dfs.namenode.http-address}}': 'c6401.ambari.apache.org:50070', - '{{cluster-env/security_enabled}}': 'false', - '{{cluster-env/smokeuser}}': 'ambari-qa', - '{{hdfs-site}}': { - 'dfs.datanode.address': '0.0.0.0:50010', - 'dfs.namenode.rpc-address': 'c6401.ambari.apache.org:8020', - 'dfs.namenode.https-address': 'c6401.ambari.apache.org:50470', - 'dfs.namenode.http-address': 'c6401.ambari.apache.org:50070', - 'dfs.datanode.https.address': '0.0.0.0:50475', - 'dfs.namenode.secondary.http-address': 'c6401.ambari.apache.org:50090', - 'dfs.datanode.http.address': '0.0.0.0:50075', - 'dfs.http.policy': 'HTTP_ONLY', - 'dfs.journalnode.https-address': '0.0.0.0:8481', - 'dfs.journalnode.http-address': '0.0.0.0:8480' - } - } global parameters parameters = { 'metric.deviation.warning.threshold': 100.0, @@ -97,6 +76,55 @@ class TestAlertMetricsDeviation(RMFTestCase): @patch("httplib.HTTPConnection") def test_alert(self, conn_mock): + configs = { + '{{hdfs-site/dfs.namenode.https-address}}': 'c6401.ambari.apache.org:50470', + '{{hdfs-site/dfs.http.policy}}': 'HTTP_ONLY', + '{{ams-site/timeline.metrics.service.webapp.address}}': 'c6401.ambari.apache.org:6188', + '{{hdfs-site/dfs.namenode.http-address}}': 'c6401.ambari.apache.org:50070', + '{{cluster-env/security_enabled}}': 'false', + '{{cluster-env/smokeuser}}': 'ambari-qa', + '{{hdfs-site}}': { + 'dfs.datanode.address': '0.0.0.0:50010', + 'dfs.namenode.rpc-address': 'c6401.ambari.apache.org:8020', + 'dfs.namenode.https-address': 'c6401.ambari.apache.org:50470', + 'dfs.namenode.http-address': 'c6401.ambari.apache.org:50070', + 'dfs.datanode.https.address': '0.0.0.0:50475', + 'dfs.namenode.secondary.http-address': 'c6401.ambari.apache.org:50090', + 'dfs.datanode.http.address': '0.0.0.0:50075', + 'dfs.http.policy': 'HTTP_ONLY', + 'dfs.journalnode.https-address': '0.0.0.0:8481', + 'dfs.journalnode.http-address': '0.0.0.0:8480' + } + } + self.make_alert_tests(configs, conn_mock) + + @patch("httplib.HTTPConnection") + def test_alert_vip(self, conn_mock): + configs = { + '{{hdfs-site/dfs.namenode.https-address}}': 'c6401.ambari.apache.org:50470', + '{{hdfs-site/dfs.http.policy}}': 'HTTP_ONLY', + '{{ams-site/timeline.metrics.service.webapp.address}}': '0.0.0.0:6188', + '{{hdfs-site/dfs.namenode.http-address}}': 'c6401.ambari.apache.org:50070', + '{{cluster-env/security_enabled}}': 'false', + '{{cluster-env/smokeuser}}': 'ambari-qa', + '{{cluster-env/metrics_collector_vip_host}}': 'c6401.ambari.apache.org', + '{{cluster-env/metrics_collector_vip_port}}': '6188', + '{{hdfs-site}}': { + 'dfs.datanode.address': '0.0.0.0:50010', + 'dfs.namenode.rpc-address': 'c6401.ambari.apache.org:8020', + 'dfs.namenode.https-address': 'c6401.ambari.apache.org:50470', + 'dfs.namenode.http-address': 'c6401.ambari.apache.org:50070', + 'dfs.datanode.https.address': '0.0.0.0:50475', + 'dfs.namenode.secondary.http-address': 'c6401.ambari.apache.org:50090', + 'dfs.datanode.http.address': '0.0.0.0:50075', + 'dfs.http.policy': 'HTTP_ONLY', + 'dfs.journalnode.https-address': '0.0.0.0:8481', + 'dfs.journalnode.http-address': '0.0.0.0:8480' + } + } + self.make_alert_tests(configs, conn_mock) + + def make_alert_tests(self, configs, conn_mock): connection = MagicMock() response = MagicMock() response.status = 200
