GuzikJakub commented on a change in pull request #4708: [AIRFLOW-3888] HA for
metastore connection
URL: https://github.com/apache/airflow/pull/4708#discussion_r262053633
##########
File path: airflow/hooks/hive_hooks.py
##########
@@ -514,20 +520,37 @@ def get_metastore_client(self):
def sasl_factory():
sasl_client = sasl.Client()
- sasl_client.setAttr("host", ms.host)
+ sasl_client.setAttr("host", valid_conn.host)
sasl_client.setAttr("service", kerberos_service_name)
sasl_client.init()
return sasl_client
from thrift_sasl import TSaslClientTransport
- transport = TSaslClientTransport(sasl_factory, "GSSAPI", socket)
+ transport = TSaslClientTransport(sasl_factory, "GSSAPI",
conn_socket)
else:
- transport = TTransport.TBufferedTransport(socket)
+ transport = TTransport.TBufferedTransport(conn_socket)
protocol = TBinaryProtocol.TBinaryProtocol(transport)
return hmsclient.HMSClient(iprot=protocol)
+ def _find_valid_server(self):
+ result = 1
+ conns = self.get_connections(self.conn_id)
+ for conn in conns:
+ host_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.log.info("Trying to connect to %s:%s", conn.host, conn.port)
+ try:
+ result = host_socket.connect_ex((conn.host, conn.port))
+ host_socket.close()
+ except Exception:
+ pass
Review comment:
Test (but I made some corrections to the code):
[2019-03-04 14:08:51,954] {hive_hooks.py:528} INFO - Trying to connect to
Metastore1
[2019-03-04 14:08:54,960] {hive_hooks.py:534} INFO - Could not connect to
Metastore1
[2019-03-04 14:08:54,960] {hive_hooks.py:528} INFO - Trying to connect to
Metastore2
[2019-03-04 14:08:54,962] {hive_hooks.py:534} INFO - Could not connect to
Metastore2
[2019-03-04 14:08:54,970] {models.py:1760} ERROR - Failed to locate the
valid server.
Traceback (most recent call last):
File "/airflow/models.py", line 1659, in _run_raw_task
result = task_copy.execute(context=context)
File "/airflow/sensors/base_sensor_operator.py", line 68, in execute
while not self.poke(context):
File "/airflow/sensors/hive_partition_sensor.py", line 73, in poke
metastore_conn_id=self.metastore_conn_id)
File "/airflow/hooks/hive_hooks.py", line 468, in __init__
self.metastore = self.get_metastore_client()
File "/airflow/hooks/hive_hooks.py", line 491, in get_metastore_client
raise AirflowException("Failed to locate the valid server.")
airflow.exceptions.AirflowException: Failed to locate the valid server.
[2019-03-04 14:08:54,971] {models.py:1789} INFO - All retries failed;
marking task as FAILED
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
