Repository: ambari Updated Branches: refs/heads/trunk 27f763450 -> 6bbed2779
AMBARI-9586 - Alerts: WEB type alerts fail in Kerberos secured cluster (Yurii Shylov via jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6bbed277 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6bbed277 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6bbed277 Branch: refs/heads/trunk Commit: 6bbed2779d05ef11294239f746abc40961fd83f7 Parents: 27f7634 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Fri Feb 13 12:08:19 2015 -0500 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Fri Feb 13 12:08:19 2015 -0500 ---------------------------------------------------------------------- .../python/ambari_agent/alerts/base_alert.py | 13 ++++- .../python/ambari_agent/alerts/web_alert.py | 55 ++++++++++++++++---- .../src/test/python/ambari_agent/TestAlerts.py | 16 +++--- .../ambari/server/state/alert/AlertUri.java | 30 +++++++++++ .../FALCON/0.5.0.2.1/alerts.json | 6 ++- .../common-services/HDFS/2.1.0.2.0/alerts.json | 12 +++-- .../common-services/OOZIE/4.0.0.2.0/alerts.json | 6 ++- .../common-services/STORM/0.9.1.2.1/alerts.json | 6 ++- .../common-services/YARN/2.1.0.2.0/alerts.json | 24 ++++++--- 9 files changed, 129 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/6bbed277/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py index 0b49b9b..2f2c997 100644 --- a/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py +++ b/ambari-agent/src/main/python/ambari_agent/alerts/base_alert.py @@ -212,6 +212,8 @@ class BaseAlert(object): https_property_key = None https_property_value_key = None default_port = None + kerberos_keytab = None + kerberos_principal = None if 'http' in uri_structure: http_key = self._find_lookup_property(uri_structure['http']) @@ -228,12 +230,19 @@ class BaseAlert(object): if 'default_port' in uri_structure: default_port = uri_structure['default_port'] + if 'kerberos_keytab' in uri_structure: + kerberos_keytab = self._find_lookup_property(uri_structure['kerberos_keytab']) + + if 'kerberos_principal' in uri_structure: + kerberos_principal = self._find_lookup_property(uri_structure['kerberos_principal']) + AlertUriLookupKeys = namedtuple('AlertUriLookupKeys', - 'http https https_property https_property_value default_port') + 'http https https_property https_property_value default_port kerberos_keytab kerberos_principal') alert_uri_lookup_keys = AlertUriLookupKeys(http=http_key, https=https_key, https_property=https_property_key, - https_property_value=https_property_value_key, default_port=default_port) + https_property_value=https_property_value_key, default_port=default_port, + kerberos_keytab=kerberos_keytab, kerberos_principal=kerberos_principal) return alert_uri_lookup_keys http://git-wip-us.apache.org/repos/asf/ambari/blob/6bbed277/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py index fa578ce..8ee6606 100644 --- a/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py +++ b/ambari-agent/src/main/python/ambari_agent/alerts/web_alert.py @@ -20,7 +20,9 @@ limitations under the License. import logging import time -import urllib2 +import subprocess +import os + from alerts.base_alert import BaseAlert from collections import namedtuple from resource_management.libraries.functions.get_port_from_url import get_port_from_url @@ -29,6 +31,8 @@ from ambari_commons.inet_utils import resolve_address logger = logging.getLogger() +CURL_CONNECTION_TIMEOUT = '20' + class WebAlert(BaseAlert): def __init__(self, alert_meta, alert_source_meta): @@ -55,9 +59,10 @@ class WebAlert(BaseAlert): web_response = self._make_web_request(url) status_code = web_response.status_code time_seconds = web_response.time_millis / 1000 + error_message = web_response.error_msg if status_code == 0: - return (self.RESULT_CRITICAL, [status_code, url, time_seconds]) + return (self.RESULT_CRITICAL, [status_code, url, time_seconds, error_message]) if status_code < 400: return (self.RESULT_OK, [status_code, url, time_seconds]) @@ -106,21 +111,49 @@ class WebAlert(BaseAlert): Makes an http(s) request to a web resource and returns the http code. If there was an error making the request, return 0 for the status code. """ - WebResponse = namedtuple('WebResponse', 'status_code time_millis') + WebResponse = namedtuple('WebResponse', 'status_code time_millis error_msg') time_millis = 0 try: - start_time = time.time() - response = urllib2.urlopen(url) + kerberos_keytab = None + kerberos_principal = None + + if self.uri_property_keys.kerberos_principal is not None: + kerberos_principal = self._lookup_property_value( + self.uri_property_keys.kerberos_principal) + + if kerberos_principal is not None: + # substitute _HOST in kerberos principal with actual fqdn + kerberos_principal = kerberos_principal.replace('_HOST', self.host_name) + + if self.uri_property_keys.kerberos_keytab is not None: + kerberos_keytab = self._lookup_property_value(self.uri_property_keys.kerberos_keytab) + + if kerberos_principal is not None and kerberos_keytab is not None: + os.system("kinit -kt {0} {1} > /dev/null".format(kerberos_keytab, kerberos_principal)) + + # substitute 0.0.0.0 in url with actual fqdn + url = url.replace('0.0.0.0', self.host_name) + start_time = time.time() + curl = subprocess.Popen(['curl', '--negotiate', '-u', ':', '-sL', '-w', + '%{http_code}', url, '--connect-timeout', CURL_CONNECTION_TIMEOUT, + '-o', '/dev/null'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + out, err = curl.communicate() + + if err != '': + raise Exception(err) + + response_code = int(out) time_millis = time.time() - start_time - except: + except Exception, exc: if logger.isEnabledFor(logging.DEBUG): - logger.exception("[Alert][{0}] Failed to make a web request".format(self.get_name())) - - return WebResponse(status_code=0, time_millis=0) - - return WebResponse(status_code=response.getcode(), time_millis=time_millis) + logger.exception("[Alert][{0}] Unable to make a web request.".format(self.get_name())) + + return WebResponse(status_code=0, time_millis=0, error_msg=str(exc)) + + return WebResponse(status_code=response_code, time_millis=time_millis, error_msg=None) def _get_reporting_text(self, state): http://git-wip-us.apache.org/repos/asf/ambari/blob/6bbed277/ambari-agent/src/test/python/ambari_agent/TestAlerts.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/test/python/ambari_agent/TestAlerts.py b/ambari-agent/src/test/python/ambari_agent/TestAlerts.py index 813f478..a04088a 100644 --- a/ambari-agent/src/test/python/ambari_agent/TestAlerts.py +++ b/ambari-agent/src/test/python/ambari_agent/TestAlerts.py @@ -419,14 +419,14 @@ class TestAlerts(TestCase): "text": "(Unit Tests) warning: {0}", }, "critical": { - "text": "(Unit Tests) critical: {1}", + "text": "(Unit Tests) critical: {1}. {3}", } } } } - WebResponse = namedtuple('WebResponse', 'status_code time_millis') - wa_make_web_request_mock.return_value = WebResponse(200,1.234) + WebResponse = namedtuple('WebResponse', 'status_code time_millis error_msg') + wa_make_web_request_mock.return_value = WebResponse(200,1.234,None) # run the alert and check HTTP 200 collector = AlertCollector() @@ -437,11 +437,11 @@ class TestAlerts(TestCase): alerts = collector.alerts() self.assertEquals(0, len(collector.alerts())) - self.assertEquals('OK', alerts[0]['state']) self.assertEquals('(Unit Tests) ok: 200', alerts[0]['text']) + self.assertEquals('OK', alerts[0]['state']) # run the alert and check HTTP 500 - wa_make_web_request_mock.return_value = WebResponse(500,1.234) + wa_make_web_request_mock.return_value = WebResponse(500,1.234,None) collector = AlertCollector() alert = WebAlert(json, json['source']) alert.set_helpers(collector, {'hdfs-site/dfs.datanode.http.address': '1.2.3.4:80'}) @@ -454,7 +454,7 @@ class TestAlerts(TestCase): self.assertEquals('(Unit Tests) warning: 500', alerts[0]['text']) # run the alert and check critical - wa_make_web_request_mock.return_value = WebResponse(0,0) + wa_make_web_request_mock.return_value = WebResponse(0,0,'error message') collector = AlertCollector() alert = WebAlert(json, json['source']) @@ -466,7 +466,7 @@ class TestAlerts(TestCase): # http assertion indicating that we properly determined non-SSL self.assertEquals('CRITICAL', alerts[0]['state']) - self.assertEquals('(Unit Tests) critical: http://1.2.3.4:80', alerts[0]['text']) + self.assertEquals('(Unit Tests) critical: http://1.2.3.4:80. error message', alerts[0]['text']) collector = AlertCollector() alert = WebAlert(json, json['source']) @@ -482,7 +482,7 @@ class TestAlerts(TestCase): # SSL assertion self.assertEquals('CRITICAL', alerts[0]['state']) - self.assertEquals('(Unit Tests) critical: https://1.2.3.4:8443', alerts[0]['text']) + self.assertEquals('(Unit Tests) critical: https://1.2.3.4:8443. error message', alerts[0]['text']) def test_reschedule(self): test_file_path = os.path.join('ambari_agent', 'dummy_files') http://git-wip-us.apache.org/repos/asf/ambari/blob/6bbed277/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java index 731e41a..fe592a8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/alert/AlertUri.java @@ -56,6 +56,18 @@ public class AlertUri { private String m_httpsPropertyValue; /** + * Kerberos keytab path to use. + */ + @SerializedName("kerberos_keytab") + private String m_kerberosKeytab; + + /** + * Kerberos principal name to use. + */ + @SerializedName("kerberos_principal") + private String m_kerberosPrincipal; + + /** * A default port to use on the host running the alert if no URLs can be * found. */ @@ -115,6 +127,24 @@ public class AlertUri { } /** + * The configuration property with kerberos keytab path. + * + * @return the configuration property, or {@code null} for none. + */ + public String getKerberosKeytab() { + return m_kerberosKeytab; + } + + /** + * The configuration property with kerberos principal name. + * + * @return the configuration property, or {@code null} for none. + */ + public String getKerberosPrincipal() { + return m_kerberosPrincipal; + } + + /** * {@inheritDoc} */ @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/6bbed277/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json index 7843676..a85787e 100644 --- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json +++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/alerts.json @@ -38,7 +38,9 @@ "type": "WEB", "uri": { "http": "{{falcon-env/falcon_port}}", - "default_port": 15000 + "default_port": 15000, + "kerberos_keytab": "{{core-site/hadoop.http.authentication.kerberos.keytab}}", + "kerberos_principal": "{{core-site/hadoop.http.authentication.kerberos.principal}}" }, "reporting": { "ok": { @@ -48,7 +50,7 @@ "text": "HTTP {0} response in {2:.3f} seconds" }, "critical": { - "text": "Connection failed to {1}" + "text": "Connection failed to {1}: {3}" } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/6bbed277/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json index 6aa7b2c..7b19415 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/alerts.json @@ -91,7 +91,9 @@ "http": "{{hdfs-site/dfs.namenode.http-address}}", "https": "{{hdfs-site/dfs.namenode.https-address}}", "https_property": "{{hdfs-site/dfs.http.policy}}", - "https_property_value": "HTTPS_ONLY" + "https_property_value": "HTTPS_ONLY", + "kerberos_keytab": "{{hdfs-site/dfs.web.authentication.kerberos.keytab}}", + "kerberos_principal": "{{hdfs-site/dfs.web.authentication.kerberos.principal}}" }, "reporting": { "ok": { @@ -101,7 +103,7 @@ "text": "HTTP {0} response in {2:.3f} seconds" }, "critical": { - "text": "Connection failed to {1}" + "text": "Connection failed to {1}: {3}" } } } @@ -482,7 +484,9 @@ "http": "{{hdfs-site/dfs.datanode.http.address}}", "https": "{{hdfs-site/dfs.datanode.https.address}}", "https_property": "{{hdfs-site/dfs.http.policy}}", - "https_property_value": "HTTPS_ONLY" + "https_property_value": "HTTPS_ONLY", + "kerberos_keytab": "{{hdfs-site/dfs.web.authentication.kerberos.keytab}}", + "kerberos_principal": "{{hdfs-site/dfs.web.authentication.kerberos.principal}}" }, "reporting": { "ok": { @@ -492,7 +496,7 @@ "text": "HTTP {0} response in {2:.3f} seconds" }, "critical": { - "text": "Connection failed to {1}" + "text": "Connection failed to {1}: {3}" } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/6bbed277/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/alerts.json b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/alerts.json index 3eae69f..a29d793 100644 --- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/alerts.json +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/alerts.json @@ -11,7 +11,9 @@ "source": { "type": "WEB", "uri": { - "http": "{{oozie-site/oozie.base.url}}/oozie" + "http": "{{oozie-site/oozie.base.url}}/oozie", + "kerberos_keytab": "{{oozie-site/oozie.authentication.kerberos.keytab}}", + "kerberos_principal": "{{oozie-site/oozie.authentication.kerberos.principal}}" }, "reporting": { "ok": { @@ -21,7 +23,7 @@ "text": "HTTP {0} response in {2:.3f} seconds" }, "critical": { - "text": "Connection failed to {1}" + "text": "Connection failed to {1}: {3}" } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/6bbed277/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json index 4c4e027..abf6b81 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/alerts.json @@ -61,7 +61,9 @@ "source": { "type": "WEB", "uri": { - "http": "{{storm-site/ui.port}}" + "http": "{{storm-site/ui.port}}", + "kerberos_keytab": "{{core-site/hadoop.http.authentication.kerberos.keytab}}", + "kerberos_principal": "{{core-site/hadoop.http.authentication.kerberos.principal}}" }, "reporting": { "ok": { @@ -71,7 +73,7 @@ "text": "HTTP {0} response in {2:.3f} seconds" }, "critical": { - "text": "Connection failed to {1}" + "text": "Connection failed to {1}: {3}" } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/6bbed277/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json index 5361bb5..efef2d0 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/alerts.json @@ -14,7 +14,9 @@ "http": "{{mapred-site/mapreduce.jobhistory.webapp.address}}", "https": "{{mapred-site/mapreduce.jobhistory.webapp.https.address}}", "https_property": "{{mapred-site/mapreduce.jobhistory.http.policy}}", - "https_property_value": "HTTPS_ONLY" + "https_property_value": "HTTPS_ONLY", + "kerberos_keytab": "{{mapred-site/mapreduce.jobhistory.webapp.spnego-keytab-file}}", + "kerberos_principal": "{{mapred-site/mapreduce.jobhistory.webapp.spnego-principal}}" }, "reporting": { "ok": { @@ -24,7 +26,7 @@ "text": "HTTP {0} response in {2:.3f} seconds" }, "critical": { - "text": "Connection failed to {1}" + "text": "Connection failed to {1}: {3}" } } } @@ -175,7 +177,9 @@ "https": "{{yarn-site/yarn.nodemanager.webapp.https.address}}", "https_property": "{{yarn-site/yarn.http.policy}}", "https_property_value": "HTTPS_ONLY", - "default_port": 8042 + "default_port": 8042, + "kerberos_keytab": "{{yarn-site/yarn.nodemanager.webapp.spnego-keytab-file}}", + "kerberos_principal": "{{yarn-site/yarn.nodemanager.webapp.spnego-principal}}" }, "reporting": { "ok": { @@ -185,7 +189,7 @@ "text": "HTTP {0} response in {2:.3f} seconds" }, "critical": { - "text": "Connection failed to {1}" + "text": "Connection failed to {1}: {3}" } } } @@ -216,7 +220,9 @@ "http": "{{yarn-site/yarn.resourcemanager.webapp.address}}", "https": "{{yarn-site/yarn.resourcemanager.webapp.https.address}}", "https_property": "{{yarn-site/yarn.http.policy}}", - "https_property_value": "HTTPS_ONLY" + "https_property_value": "HTTPS_ONLY", + "kerberos_keytab": "{{yarn-site/yarn.resourcemanager.webapp.spnego-keytab-file}}", + "kerberos_principal": "{{yarn-site/yarn.resourcemanager.webapp.spnego-principal}}" }, "reporting": { "ok": { @@ -226,7 +232,7 @@ "text": "HTTP {0} response in {2:.3f} seconds" }, "critical": { - "text": "Connection failed to {1}" + "text": "Connection failed to {1}: {3}" } } } @@ -333,7 +339,9 @@ "http": "{{yarn-site/yarn.timeline-service.webapp.address}}", "https": "{{yarn-site/yarn.timeline-service.webapp.https.address}}", "https_property": "{{yarn-site/yarn.http.policy}}", - "https_property_value": "HTTPS_ONLY" + "https_property_value": "HTTPS_ONLY", + "kerberos_keytab": "{{yarn-site/yarn.timeline-service.http-authentication.kerberos.keytab}}", + "kerberos_principal": "{{yarn-site/yarn.timeline-service.http-authentication.kerberos.principal}}" }, "reporting": { "ok": { @@ -343,7 +351,7 @@ "text": "HTTP {0} response in {2:.3f} seconds" }, "critical": { - "text": "Connection failed to {1}" + "text": "Connection failed to {1}: {3}" } } }
