Repository: ambari Updated Branches: refs/heads/branch-1.7.0 5bb80bd8e -> 55b60c4d4 refs/heads/trunk 9d7f877f4 -> d9c383891
AMBARI-7733. Alerts on Hiveserver2 (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d9c38389 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d9c38389 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d9c38389 Branch: refs/heads/trunk Commit: d9c3838913f8c9f959c3543d4898483cb851f877 Parents: 9d7f877 Author: Andrew Onishuk <[email protected]> Authored: Fri Oct 10 19:45:46 2014 +0300 Committer: Andrew Onishuk <[email protected]> Committed: Fri Oct 10 19:45:46 2014 +0300 ---------------------------------------------------------------------- .../libraries/functions/hive_check.py | 9 +++++++-- ambari-common/src/main/unix/ambari-python-wrap | 2 +- .../1.3.2/services/HIVE/package/scripts/hive_service.py | 2 +- .../services/HIVE/package/scripts/service_check.py | 5 +++-- .../NAGIOS/package/files/check_hive_thrift_port.py | 12 +++++++++--- .../NAGIOS/package/templates/hadoop-commands.cfg.j2 | 2 +- .../NAGIOS/package/templates/hadoop-services.cfg.j2 | 2 +- .../2.0.6/services/HIVE/package/scripts/hive_service.py | 2 +- .../services/HIVE/package/scripts/service_check.py | 5 +++-- .../NAGIOS/package/files/check_hive_thrift_port.py | 12 +++++++++--- .../NAGIOS/package/templates/hadoop-commands.cfg.j2 | 2 +- .../NAGIOS/package/templates/hadoop-services.cfg.j2 | 2 +- .../python/stacks/1.3.2/HIVE/test_hive_service_check.py | 8 +++++--- .../python/stacks/2.0.6/HIVE/test_hive_service_check.py | 9 +++++---- 14 files changed, 48 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py b/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py index c5f557e..282fed2 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py @@ -20,7 +20,7 @@ limitations under the License. import socket from resource_management.core.exceptions import Fail -def check_thrift_port_sasl(address, port, timeout = 5): +def check_thrift_port_sasl(address, port, timeout = 5, security_enabled = False): """ Hive thrift SASL port check """ @@ -73,8 +73,13 @@ def check_thrift_port_sasl(address, port, timeout = 5): try: s.connect((address, port)) - s.send(msg) + #Successfull connection, port check passed is_service_socket_valid = True + + # Try to send anonymous plain auth message to thrift to prevent errors in hive log + # Plain mechanism is not supported in security mode + if not security_enabled: + s.send(msg) except socket.error, e: #Expected if service unreachable pass http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-common/src/main/unix/ambari-python-wrap ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/unix/ambari-python-wrap b/ambari-common/src/main/unix/ambari-python-wrap index ef19a0f..b9cf3b6 100755 --- a/ambari-common/src/main/unix/ambari-python-wrap +++ b/ambari-common/src/main/unix/ambari-python-wrap @@ -17,7 +17,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -export PYTHONPATH=/usr/lib/python2.6/site-packages/ambari_commons:$PYTHONPATH +export PYTHONPATH=/usr/lib/python2.6/site-packages:/usr/lib/python2.6/site-packages/ambari_commons:$PYTHONPATH # reset settings unset PYTHON http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py index fa11b28..d3bd71b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py @@ -69,7 +69,7 @@ def hive_service( is_service_socket_valid = False print "Waiting for the Hive server to start..." while time.time() < end_time: - if check_thrift_port_sasl(address, port, 2): + if check_thrift_port_sasl(address, port, 2, security_enabled=params.security_enabled): is_service_socket_valid = True break else: http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py index e6d5494..b75578b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py @@ -33,10 +33,11 @@ class HiveServiceCheck(Script): address=format("{hive_server_host}") port=int(format("{hive_server_port}")) print "Test connectivity to hive server" - if check_thrift_port_sasl(address, port): + if check_thrift_port_sasl(address, port, security_enabled=params.security_enabled): print "Successfully connected to %s on port %s" % (address, port) else: - print "Connection to %s on port %s failed: %s" % (address, port) + print "Connection to %s on port %s failed" % (address, port) + exit(1) hcat_service_check() webhcat_service_check() http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py index cd39641..c9414f7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py @@ -37,22 +37,28 @@ def main(): parser.add_option("-H", "--host", dest="address", help="Hive thrift host") parser.add_option("-p", "--port", type="int", dest="port", help="Hive thrift port") + parser.add_option("--security-enabled", action="store_true", dest="security_enabled") (options, args) = parser.parse_args() if options.address is None: - print "Hive thrift host (--name or -n)" + print "Specify hive thrift host (--host or -H)" exit(-1) if options.port is None: - print "Hive thrift port (--file or -f)" + print "Specify hive thrift port (--port or -p)" exit(-1) + if options.security_enabled: + security_enabled = options.security_enabled + else: + security_enabled = False + address = options.address port = options.port starttime = time() - if check_thrift_port_sasl(address, port): + if check_thrift_port_sasl(address, port, security_enabled=security_enabled): timetaken = time() - starttime print OK_MESSAGE % (timetaken, port) exit(0) http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 index 8f5a230..a81762f 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 @@ -143,5 +143,5 @@ define command{ define command{ command_name check_tcp_wrapper_sasl - command_line $USER1$/check_wrapper.sh /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H $HOSTADDRESS$ -p $ARG1$ + command_line $USER1$/check_wrapper.sh /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H $HOSTADDRESS$ -p $ARG1$ $ARG2$ } http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2 index 5fd0777..6ce5d10 100644 --- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2 @@ -539,7 +539,7 @@ define service { use hadoop-service service_description HIVE-SERVER::HiveServer2 process servicegroups HIVE - check_command check_tcp_wrapper_sasl!{{ hive_server_port }}!-w 1 -c 1 + check_command check_tcp_wrapper_sasl!{{ hive_server_port }}!{{ '--security-enabled' if security_enabled else '' }}!-w 1 -c 1 normal_check_interval 0.5 retry_check_interval 0.5 max_check_attempts 3 http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py index e40e9f8..5463df4 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py @@ -75,7 +75,7 @@ def hive_service( is_service_socket_valid = False print "Waiting for the Hive server to start..." while time.time() < end_time: - if check_thrift_port_sasl(address, port, 2): + if check_thrift_port_sasl(address, port, 2, security_enabled=params.security_enabled): is_service_socket_valid = True break else: http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py index e6d5494..b75578b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py @@ -33,10 +33,11 @@ class HiveServiceCheck(Script): address=format("{hive_server_host}") port=int(format("{hive_server_port}")) print "Test connectivity to hive server" - if check_thrift_port_sasl(address, port): + if check_thrift_port_sasl(address, port, security_enabled=params.security_enabled): print "Successfully connected to %s on port %s" % (address, port) else: - print "Connection to %s on port %s failed: %s" % (address, port) + print "Connection to %s on port %s failed" % (address, port) + exit(1) hcat_service_check() webhcat_service_check() http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py index cd39641..c9414f7 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py @@ -37,22 +37,28 @@ def main(): parser.add_option("-H", "--host", dest="address", help="Hive thrift host") parser.add_option("-p", "--port", type="int", dest="port", help="Hive thrift port") + parser.add_option("--security-enabled", action="store_true", dest="security_enabled") (options, args) = parser.parse_args() if options.address is None: - print "Hive thrift host (--name or -n)" + print "Specify hive thrift host (--host or -H)" exit(-1) if options.port is None: - print "Hive thrift port (--file or -f)" + print "Specify hive thrift port (--port or -p)" exit(-1) + if options.security_enabled: + security_enabled = options.security_enabled + else: + security_enabled = False + address = options.address port = options.port starttime = time() - if check_thrift_port_sasl(address, port): + if check_thrift_port_sasl(address, port, security_enabled=security_enabled): timetaken = time() - starttime print OK_MESSAGE % (timetaken, port) exit(0) http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 index 656ee7f..39bb636 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2 @@ -157,7 +157,7 @@ define command{ define command{ command_name check_tcp_wrapper_sasl - command_line $USER1$/check_wrapper.sh /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H $HOSTADDRESS$ -p $ARG1$ + command_line $USER1$/check_wrapper.sh /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H $HOSTADDRESS$ -p $ARG1$ $ARG2$ } define command{ http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 index fd15e6b..f69238e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2 @@ -700,7 +700,7 @@ define service { use hadoop-service service_description HIVE-SERVER::HiveServer2 process servicegroups HIVE - check_command check_tcp_wrapper_sasl!{{ hive_server_port }}!-w 1 -c 1 + check_command check_tcp_wrapper_sasl!{{ hive_server_port }}!{{ '--security-enabled' if security_enabled else '' }}!-w 1 -c 1 normal_check_interval 0.5 retry_check_interval 0.5 max_check_attempts 3 http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py index 449c950..ad75ce1 100644 --- a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py +++ b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py @@ -22,11 +22,11 @@ from stacks.utils.RMFTestCase import * import datetime, socket import resource_management.libraries.functions @patch.object(resource_management.libraries.functions, "get_unique_id_and_date", new = MagicMock(return_value='')) -@patch("socket.socket", new = MagicMock()) +@patch("socket.socket") class TestServiceCheck(RMFTestCase): @patch("sys.exit") - def test_service_check_default(self, sys_exit_mock): + def test_service_check_default(self, sys_exit_mock, socket_mock): self.executeScript("1.3.2/services/HIVE/package/scripts/service_check.py", classname="HiveServiceCheck", @@ -70,9 +70,10 @@ class TestServiceCheck(RMFTestCase): try_sleep = 5, ) self.assertNoMoreResources() + self.assertTrue(socket_mock.called) @patch("sys.exit") - def test_service_check_secured(self, sys_exit_mock): + def test_service_check_secured(self, sys_exit_mock, socket_mock): self.executeScript("1.3.2/services/HIVE/package/scripts/service_check.py", classname="HiveServiceCheck", @@ -117,3 +118,4 @@ class TestServiceCheck(RMFTestCase): try_sleep = 5, ) self.assertNoMoreResources() + self.assertTrue(socket_mock.called) http://git-wip-us.apache.org/repos/asf/ambari/blob/d9c38389/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py index 01da905..6ba4d48 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py +++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py @@ -23,11 +23,11 @@ from stacks.utils.RMFTestCase import * import datetime, sys, socket import resource_management.libraries.functions @patch.object(resource_management.libraries.functions, "get_unique_id_and_date", new = MagicMock(return_value='')) -@patch("socket.socket", new = MagicMock()) +@patch("socket.socket") class TestServiceCheck(RMFTestCase): @patch("sys.exit") - def test_service_check_default(self, sys_exit_mock): + def test_service_check_default(self, sys_exit_mock, socket_mock): self.executeScript("2.0.6/services/HIVE/package/scripts/service_check.py", classname="HiveServiceCheck", @@ -72,9 +72,10 @@ class TestServiceCheck(RMFTestCase): try_sleep = 5, ) self.assertNoMoreResources() + self.assertTrue(socket_mock.called) @patch("sys.exit") - def test_service_check_secured(self, sys_exit_mock): + def test_service_check_secured(self, sys_exit_mock, socket_mock): self.executeScript("2.0.6/services/HIVE/package/scripts/service_check.py", classname="HiveServiceCheck", @@ -121,4 +122,4 @@ class TestServiceCheck(RMFTestCase): try_sleep = 5, ) self.assertNoMoreResources() - self.assertNoMoreResources() + self.assertTrue(socket_mock.called)
