mik-laj commented on a change in pull request #4064: AIRFLOW-3149 Support
dataproc cluster deletion on ERROR
URL: https://github.com/apache/airflow/pull/4064#discussion_r253279580
##########
File path: airflow/contrib/operators/dataproc_operator.py
##########
@@ -238,30 +238,67 @@ def _get_cluster(self, service):
def _get_cluster_state(self, service):
cluster = self._get_cluster(service)
- if 'status' in cluster:
+ if cluster and 'status' in cluster:
return cluster['status']['state']
else:
return None
def _cluster_ready(self, state, service):
if state == 'RUNNING':
return True
+ if state == 'DELETING':
+ raise Exception('Tried to create a cluster but it\'s in DELETING,
something went wrong.')
if state == 'ERROR':
cluster = self._get_cluster(service)
try:
error_details = cluster['status']['details']
except KeyError:
error_details = 'Unknown error in cluster creation, ' \
'check Google Cloud console for details.'
+
+ self.log.info('Dataproc cluster creation resulted in an ERROR
state running diagnostics')
+ self.log.info(error_details)
+ diagnose_operation_name = \
+
DataprocClusterCreateOperator._execute_dataproc_diagnose(service,
self.project_id,
+
self.region, self.cluster_name)
+ diagnose_result = DataProcHook._wait_for_operation_done(service,
diagnose_operation_name)
+ if diagnose_result.get('response') and
diagnose_result.get('response').get('outputUri'):
+ self.log.info('Diagnostic information for ERROR cluster
available at [' +
Review comment:
```suggestion
output_url .=
diagnose_result.get('response').get('outputUri')
self.log.info('Diagnostic information for ERROR cluster
available at [%s]', output_url)
```
You should avoid formatting the text before passing it to the logger. When
text and data are transferred to the logger separately, special loggers allow
to analyze in a deeper way.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
