Repository: ambari Updated Branches: refs/heads/trunk c22e1ec08 -> a84acc27f
AMBARI-9096. Cluster create fails when "hive.server2.transport.mode" property is set to "http" (dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a84acc27 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a84acc27 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a84acc27 Branch: refs/heads/trunk Commit: a84acc27f703e3e74e33e8022c0af6df6838b69a Parents: c22e1ec Author: Lisnichenko Dmitro <[email protected]> Authored: Tue Jan 13 22:24:26 2015 +0200 Committer: Lisnichenko Dmitro <[email protected]> Committed: Tue Jan 13 22:24:26 2015 +0200 ---------------------------------------------------------------------- .../libraries/functions/hive_check.py | 5 ++-- .../package/alerts/alert_hive_thrift_port.py | 28 +++++++++++++------- .../0.12.0.2.0/package/scripts/hive_service.py | 3 ++- .../HIVE/0.12.0.2.0/package/scripts/params.py | 6 ++++- .../0.12.0.2.0/package/scripts/service_check.py | 3 ++- .../python/stacks/2.0.6/configs/default.json | 1 + .../stacks/2.0.6/configs/default_client.json | 3 ++- .../python/stacks/2.0.6/configs/secured.json | 3 ++- .../stacks/2.0.6/configs/secured_client.json | 3 ++- .../test/python/stacks/2.1/configs/default.json | 3 ++- .../test/python/stacks/2.1/configs/secured.json | 3 ++- 11 files changed, 42 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py b/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py index 1924c05..f0036be 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py @@ -22,7 +22,8 @@ from resource_management.core.resources import Execute from resource_management.libraries.functions import format import socket -def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kinitcmd = None, smokeuser = 'ambari-qa'): +def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kinitcmd = None, smokeuser = 'ambari-qa', + transport_mode = "binary"): """ Hive thrift SASL port check """ @@ -36,7 +37,7 @@ def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kini else: url = format("jdbc:hive2://{address}:{port}") - if hive_auth != "NOSASL": + if hive_auth != "NOSASL" and transport_mode != "http": cmd = format("! beeline -u '{url}' -e '' ") + "2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL'" Execute(cmd, user=smokeuser, http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py index ebfbf55..9979530 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py @@ -31,6 +31,8 @@ OK_MESSAGE = "TCP OK - %.4f response on port %s" CRITICAL_MESSAGE = "Connection failed on host {0}:{1}" HIVE_SERVER_THRIFT_PORT_KEY = '{{hive-site/hive.server2.thrift.port}}' +HIVE_SERVER_THRIFT_HTTP_PORT_KEY = '{{hive-site/hive.server2.thrift.http.port}}' +HIVE_SERVER_TRANSPORT_MODE_KEY = '{{hive-site/hive.server2.transport.mode}}' SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}' HIVE_SERVER2_AUTHENTICATION_KEY = '{{hive-site/hive.server2.authentication}}' HIVE_SERVER_PRINCIPAL_KEY = '{{hive-site/hive.server2.authentication.kerberos.principal}}' @@ -41,6 +43,7 @@ PERCENT_WARNING = 200 PERCENT_CRITICAL = 200 THRIFT_PORT_DEFAULT = 10000 +HIVE_SERVER_TRANSPORT_MODE_DEFAULT = 'binary' HIVE_SERVER_PRINCIPAL_DEFAULT = 'hive/[email protected]' HIVE_SERVER2_AUTHENTICATION_DEFAULT = 'NOSASL' SMOKEUSER_KEYTAB_DEFAULT = '/etc/security/keytabs/smokeuser.headless.keytab' @@ -51,7 +54,8 @@ def get_tokens(): Returns a tuple of tokens in the format {{site/property}} that will be used to build the dictionary passed into execute """ - return (HIVE_SERVER_THRIFT_PORT_KEY,SECURITY_ENABLED_KEY,HIVE_SERVER2_AUTHENTICATION_KEY,HIVE_SERVER_PRINCIPAL_KEY,SMOKEUSER_KEYTAB_KEY,SMOKEUSER_KEY) + return (HIVE_SERVER_THRIFT_PORT_KEY,SECURITY_ENABLED_KEY,HIVE_SERVER2_AUTHENTICATION_KEY,HIVE_SERVER_PRINCIPAL_KEY, + SMOKEUSER_KEYTAB_KEY,SMOKEUSER_KEY,HIVE_SERVER_THRIFT_HTTP_PORT_KEY,HIVE_SERVER_TRANSPORT_MODE_KEY) def execute(parameters=None, host_name=None): @@ -66,9 +70,15 @@ def execute(parameters=None, host_name=None): if parameters is None: return (('UNKNOWN', ['There were no parameters supplied to the script.'])) - thrift_port = THRIFT_PORT_DEFAULT - if HIVE_SERVER_THRIFT_PORT_KEY in parameters: - thrift_port = int(parameters[HIVE_SERVER_THRIFT_PORT_KEY]) + transport_mode = HIVE_SERVER_TRANSPORT_MODE_DEFAULT + if HIVE_SERVER_TRANSPORT_MODE_KEY in parameters: + transport_mode = parameters[HIVE_SERVER_TRANSPORT_MODE_KEY] + + port = THRIFT_PORT_DEFAULT + if transport_mode.lower() == 'binary' and HIVE_SERVER_THRIFT_PORT_KEY in parameters: + port = int(parameters[HIVE_SERVER_THRIFT_PORT_KEY]) + elif transport_mode.lower() == 'http' and HIVE_SERVER_THRIFT_HTTP_PORT_KEY in parameters: + port = int(parameters[HIVE_SERVER_THRIFT_HTTP_PORT_KEY]) security_enabled = False if SECURITY_ENABLED_KEY in parameters: @@ -103,8 +113,8 @@ def execute(parameters=None, host_name=None): start_time = time.time() try: - hive_check.check_thrift_port_sasl(host_name, thrift_port, hive_server2_authentication, - hive_server_principal, kinitcmd, smokeuser) + hive_check.check_thrift_port_sasl(host_name, port, hive_server2_authentication, + hive_server_principal, kinitcmd, smokeuser, transport_mode = transport_mode) is_thrift_port_ok = True except: is_thrift_port_ok = False @@ -112,13 +122,13 @@ def execute(parameters=None, host_name=None): if is_thrift_port_ok == True: result_code = 'OK' total_time = time.time() - start_time - label = OK_MESSAGE % (total_time, thrift_port) + label = OK_MESSAGE % (total_time, port) else: result_code = 'CRITICAL' - label = CRITICAL_MESSAGE.format(host_name,thrift_port) + label = CRITICAL_MESSAGE.format(host_name,port) except Exception, e: label = str(e) result_code = 'UNKNOWN' - return ((result_code, [label])) \ No newline at end of file + return ((result_code, [label])) http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py index 1120b5d..3b32977 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py @@ -82,7 +82,8 @@ def hive_service(name, action='start', rolling_restart=False): while time.time() < end_time: try: check_thrift_port_sasl(address, port, params.hive_server2_authentication, - params.hive_server_principal, kinitcmd, params.smokeuser) + params.hive_server_principal, kinitcmd, params.smokeuser, + transport_mode=params.hive_transport_mode) is_service_socket_valid = True break except Exception, e: http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py index a027d52..00bada8 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py @@ -135,7 +135,11 @@ hive_var_lib = '/var/lib/hive' ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0] hive_server_host = config['clusterHostInfo']['hive_server_host'][0] hive_server_hosts = config['clusterHostInfo']['hive_server_host'] -hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000") +hive_transport_mode = config['configurations']['hive-site']['hive.server2.transport.mode'] +if hive_transport_mode.lower() == "http": + hive_server_port = config['configurations']['hive-site']['hive.server2.thrift.http.port'] +else: + hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000") hive_url = format("jdbc:hive2://{hive_server_host}:{hive_server_port}") hive_server_principal = config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal'] hive_server2_authentication = config['configurations']['hive-site']['hive.server2.authentication'] http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py index 4e8b886..09d31aa 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py @@ -42,7 +42,8 @@ class HiveServiceCheck(Script): for address in address_list: try: check_thrift_port_sasl(address, port, params.hive_server2_authentication, - params.hive_server_principal, kinitcmd, params.smokeuser) + params.hive_server_principal, kinitcmd, params.smokeuser, + transport_mode=params.hive_transport_mode) print "Successfully connected to %s on port %s" % (address, port) workable_server_available = True except: http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.0.6/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json index 82b485b..9132b2f 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json @@ -370,6 +370,7 @@ "hive.auto.convert.sortmerge.join.noconditionaltask": "true", "hive.server2.enable.doAs": "true", "hive.server2.authentication": "NOSASL", + "hive.server2.transport.mode": "binary", "hive.optimize.mapjoin.mapreduce": "true" }, "yarn-site": { http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json index f544b88..8e643de 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json @@ -364,7 +364,8 @@ "ambari.hive.db.schema.name": "hive", "hive.metastore.execute.setugi": "true", "hive.auto.convert.sortmerge.join.noconditionaltask": "true", - "hive.server2.enable.doAs": "true", + "hive.server2.enable.doAs": "true", + "hive.server2.transport.mode": "binary", "hive.optimize.mapjoin.mapreduce": "true" }, "yarn-site": { http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json index d48b0ab..119ab88 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json @@ -407,7 +407,8 @@ "hive.metastore.execute.setugi": "true", "hive.auto.convert.sortmerge.join.noconditionaltask": "true", "hive.server2.enable.doAs": "true", - "hive.optimize.mapjoin.mapreduce": "true", + "hive.optimize.mapjoin.mapreduce": "true", + "hive.server2.transport.mode": "binary", "hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab" }, "yarn-site": { http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json index 69c7b4d..4faf0f8 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json +++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json @@ -405,7 +405,8 @@ "hive.metastore.execute.setugi": "true", "hive.auto.convert.sortmerge.join.noconditionaltask": "true", "hive.server2.enable.doAs": "true", - "hive.optimize.mapjoin.mapreduce": "true", + "hive.optimize.mapjoin.mapreduce": "true", + "hive.server2.transport.mode": "binary", "hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab" }, "yarn-site": { http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.1/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/configs/default.json b/ambari-server/src/test/python/stacks/2.1/configs/default.json index ac829dd..689279d 100644 --- a/ambari-server/src/test/python/stacks/2.1/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.1/configs/default.json @@ -479,7 +479,8 @@ "ambari.hive.db.schema.name": "hive", "hive.metastore.execute.setugi": "true", "hive.auto.convert.sortmerge.join.noconditionaltask": "true", - "hive.server2.enable.doAs": "true", + "hive.server2.enable.doAs": "true", + "hive.server2.transport.mode": "binary", "hive.optimize.mapjoin.mapreduce": "true" }, "yarn-site": { http://git-wip-us.apache.org/repos/asf/ambari/blob/a84acc27/ambari-server/src/test/python/stacks/2.1/configs/secured.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.1/configs/secured.json b/ambari-server/src/test/python/stacks/2.1/configs/secured.json index 0ba831d..b2a6053 100644 --- a/ambari-server/src/test/python/stacks/2.1/configs/secured.json +++ b/ambari-server/src/test/python/stacks/2.1/configs/secured.json @@ -451,7 +451,8 @@ "hive.metastore.execute.setugi": "true", "hive.auto.convert.sortmerge.join.noconditionaltask": "true", "hive.server2.enable.doAs": "true", - "hive.optimize.mapjoin.mapreduce": "true", + "hive.optimize.mapjoin.mapreduce": "true", + "hive.server2.transport.mode": "binary", "hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab" }, "yarn-site": {
