syedahsn commented on code in PR #30244:
URL: https://github.com/apache/airflow/pull/30244#discussion_r1163043125
##########
airflow/providers/amazon/aws/hooks/redshift_cluster.py:
##########
@@ -285,6 +285,47 @@ async def resume_cluster(
except botocore.exceptions.ClientError as error:
return {"status": "error", "message": str(error)}
+ async def delete_cluster(
+ self,
+ cluster_identifier: str,
+ skip_final_cluster_snapshot: bool = True,
+ final_cluster_snapshot_identifier: str | None = None,
+ poll_interval: float = 5.0,
+ ) -> dict[str, Any]:
+ """
+ Connects to the AWS redshift cluster via aiobotocore and
+ deletes the cluster based on the cluster_identifier passed
+
+ :param cluster_identifier: unique identifier of a cluster
+ :param skip_final_cluster_snapshot: determines cluster snapshot
creation
+ :param final_cluster_snapshot_identifier: name of final cluster
snapshot
+ :param poll_interval: polling period in seconds to check for the status
+ """
+ try:
+ final_cluster_snapshot_identifier =
final_cluster_snapshot_identifier or ""
+
+ async with await self.get_client_async() as client:
+ response = await client.delete_cluster(
+ ClusterIdentifier=cluster_identifier,
+ SkipFinalClusterSnapshot=skip_final_cluster_snapshot,
+
FinalClusterSnapshotIdentifier=final_cluster_snapshot_identifier,
+ )
+ status = response["Cluster"]["ClusterStatus"] if response and
response["Cluster"] else None
+ if status == "deleting":
Review Comment:
If the initial call to `delete_cluster` hits the `InvalidClusterStateFault`
Exception, then this function will not attempt to retry. The
`InvalidClusterStateFault` is only seen in the first few attempts, when the
cluster is not ready to accept new instructions even if it is reporting its
state as `available`.
##########
airflow/providers/amazon/aws/triggers/redshift_cluster.py:
##########
@@ -80,8 +87,25 @@ async def run(self) -> AsyncIterator["TriggerEvent"]:
else:
error_message = f"{self.task_id} failed"
yield TriggerEvent({"status": "error", "message":
error_message})
+ elif self.operation_type == "delete_cluster":
+ response = await hook.delete_cluster(
+ cluster_identifier=self.cluster_identifier,
+
skip_final_cluster_snapshot=self.skip_final_cluster_snapshot,
+
final_cluster_snapshot_identifier=self.final_cluster_snapshot_identifier,
+ poll_interval=self.poll_interval,
+ )
+ if response.get("status") == "success":
+ yield TriggerEvent(response)
+ else:
+ if self.attempts < 1:
+ error_message = f"{self.task_id} failed"
+ yield TriggerEvent({"status": "error", "message":
error_message})
else:
yield TriggerEvent(f"{self.operation_type} is not
supported")
+ self.log.info(
+ "Retrying in #%s seconds remaining attempts #%s",
self.poll_interval, self.attempts
+ )
+ await asyncio.sleep(self.poll_interval)
except Exception as e:
if self.attempts < 1:
yield TriggerEvent({"status": "error", "message": str(e)})
Review Comment:
This is still not correct I think. If one of the operations from the `try`
block above raise an exception, it will get caught over here. Then depending on
the number of attempts left (which may be more 1) a `TriggerEvent` will be
yielded. Otherwise the exception will be silenced.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
