nsAstro commented on code in PR #29732:
URL: https://github.com/apache/airflow/pull/29732#discussion_r1124519620
##########
airflow/providers/amazon/aws/hooks/emr.py:
##########
@@ -68,22 +68,24 @@ def get_cluster_id_by_name(self, emr_cluster_name: str,
cluster_states: list[str
:param emr_cluster_name: Name of a cluster to find
:param cluster_states: State(s) of cluster to find
:return: id of the EMR cluster
- """
+ """
response = self.get_conn().list_clusters(ClusterStates=cluster_states)
+ while true:
+ matching_clusters = list(
+ filter(lambda cluster: cluster["Name"] == emr_cluster_name,
response["Clusters"])
+ )
- matching_clusters = list(
- filter(lambda cluster: cluster["Name"] == emr_cluster_name,
response["Clusters"])
- )
-
- if len(matching_clusters) == 1:
- cluster_id = matching_clusters[0]["Id"]
- self.log.info("Found cluster name = %s id = %s", emr_cluster_name,
cluster_id)
- return cluster_id
- elif len(matching_clusters) > 1:
- raise AirflowException(f"More than one cluster found for name
{emr_cluster_name}")
- else:
- self.log.info("No cluster found for name %s", emr_cluster_name)
- return None
+ if len(matching_clusters) == 1:
+ cluster_id = matching_clusters[0]["Id"]
+ self.log.info("Found cluster name = %s id = %s",
emr_cluster_name, cluster_id)
+ return cluster_id
+ elif len(matching_clusters) > 1:
+ raise AirflowException(f"More than one cluster found for name
{emr_cluster_name}")
+ elif response["Marker"]:
Review Comment:
Yeah, I was optimizing for speed when I thought we were filtering on a
unique ID. Just rewrote to use the above.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
