yuseok89 commented on code in PR #74602:
URL: https://github.com/apache/airflow/pull/74602#discussion_r4255271069
##########
providers/google/src/airflow/providers/google/cloud/triggers/dataproc.py:
##########
@@ -651,13 +653,32 @@ async def run(self):
hook = self.get_async_hook()
while True:
- batch = await hook.get_batch(
- project_id=self.project_id, region=self.region,
batch_id=self.batch_id
- )
+ try:
+ batch = await hook.get_batch(
+ project_id=self.project_id,
+ region=self.region,
+ batch_id=self.batch_id,
+ )
+ except ServiceUnavailable as e:
+ self.log.warning("Dataproc API is temporarily unavailable.
Retrying... (%s)", e)
+ await asyncio.sleep(self.polling_interval_seconds)
+ continue
+ except AioRpcError as e:
+ if e.code() != grpc.StatusCode.UNAVAILABLE:
+ raise
+ self.log.warning("Dataproc gRPC UNAVAILABLE. Retrying...
(%s)", e)
+ await asyncio.sleep(self.polling_interval_seconds)
+ continue
+
state = batch.state
- if state in (Batch.State.FAILED, Batch.State.SUCCEEDED,
Batch.State.CANCELLED):
+ if state in (
+ Batch.State.FAILED,
+ Batch.State.SUCCEEDED,
+ Batch.State.CANCELLED,
+ ):
break
+
Review Comment:
nit: the reformatting isn't related to the fix, so it could be left as it
was to keep the diff focused.
##########
providers/google/src/airflow/providers/google/cloud/triggers/dataproc.py:
##########
@@ -651,13 +653,32 @@ async def run(self):
hook = self.get_async_hook()
while True:
- batch = await hook.get_batch(
- project_id=self.project_id, region=self.region,
batch_id=self.batch_id
- )
+ try:
+ batch = await hook.get_batch(
+ project_id=self.project_id,
+ region=self.region,
+ batch_id=self.batch_id,
+ )
+ except ServiceUnavailable as e:
+ self.log.warning("Dataproc API is temporarily unavailable.
Retrying... (%s)", e)
+ await asyncio.sleep(self.polling_interval_seconds)
+ continue
+ except AioRpcError as e:
Review Comment:
Can this branch actually be reached?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
