uranusjr commented on a change in pull request #19711:
URL: https://github.com/apache/airflow/pull/19711#discussion_r755367602
##########
File path: airflow/providers/apache/hdfs/hooks/webhdfs.py
##########
@@ -71,30 +71,31 @@ def get_conn(self) -> Any:
def _find_valid_server(self) -> Any:
connection = self.get_connection(self.webhdfs_conn_id)
- host_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- self.log.info("Trying to connect to %s:%s", connection.host,
connection.port)
- try:
- conn_check = host_socket.connect_ex((connection.host,
connection.port))
- if conn_check == 0:
- self.log.info('Trying namenode %s', connection.host)
- client = self._get_client(connection)
- client.status('/')
- self.log.info('Using namenode %s for hook', connection.host)
- host_socket.close()
- return client
- else:
- self.log.error("Could not connect to %s:%s", connection.host,
connection.port)
- host_socket.close()
- except HdfsError as hdfs_error:
- self.log.error('Read operation on namenode %s failed with error:
%s', connection.host, hdfs_error)
+ namenodes = connection.host.split(',')
+ for namenode in namenodes:
+ host_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.log.info("Trying to connect to %s:%s", namenode,
connection.port)
+ try:
+ conn_check = host_socket.connect_ex((namenode,
connection.port))
+ if conn_check == 0:
+ self.log.info('Trying namenode %s', namenode)
+ client = self._get_client(connection, namenode)
+ client.status('/')
+ self.log.info('Using namenode %s for hook', namenode)
+ host_socket.close()
+ return client
+ else:
+ self.log.error("Could not connect to %s:%s", namenode,
connection.port)
Review comment:
If only one connection is needed, would it be better to have this as a
`info` log instead, or even lower?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
