nevcohen commented on code in PR #61110:
URL: https://github.com/apache/airflow/pull/61110#discussion_r2742348972
##########
providers/cncf/kubernetes/tests/unit/cncf/kubernetes/operators/test_spark_kubernetes.py:
##########
@@ -983,23 +991,24 @@ def test_find_spark_job_picks_running_pod(
running_pod.metadata.labels = {"try_number": "1"}
running_pod.status.phase = "Running"
- # Pending pod should not be selected.
- pending_pod = mock.MagicMock()
- pending_pod.metadata.creation_timestamp = timezone.datetime(2025, 1,
1, tzinfo=timezone.utc)
- pending_pod.metadata.name = "spark-driver-pending"
- pending_pod.metadata.labels = {"try_number": "1"}
- pending_pod.status.phase = "Pending"
+ # Terminating pod should not be selected.
+ terminating_pod = mock.MagicMock()
+ terminating_pod.metadata.creation_timestamp = timezone.datetime(2025,
1, 1, tzinfo=timezone.utc)
+ terminating_pod.metadata.deletion_timestamp = timezone.datetime(2025,
1, 2, tzinfo=timezone.utc)
+ terminating_pod.metadata.name = "spark-driver-pending"
+ terminating_pod.metadata.labels = {"try_number": "1"}
+ terminating_pod.status.phase = "Running"
mock_get_kube_client.list_namespaced_pod.return_value.items = [
running_pod,
- pending_pod,
+ terminating_pod,
]
returned_pod = op.find_spark_job(context)
assert returned_pod is running_pod
- def test_find_spark_job_picks_latest_pod(
+ def test_find_spark_job_picks_pending_pod(
Review Comment:
In the current logic it will also return a spark job that completed
successfully assuming there is one of it, correct me if I'm wrong?
##########
providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py:
##########
@@ -279,6 +283,9 @@ def find_spark_job(self, context, exclude_checked: bool =
True):
self.log.info("`try_number` of pod: %s",
pod.metadata.labels.get("try_number", "unknown"))
return pod
+ def _get_field_selector(self) -> str:
+ return
f"status.phase!={PodPhase.SUCCEEDED},status.phase!={PodPhase.FAILED},status.phase!={PodPhase.UNKNOWN}"
Review Comment:
What happens if the driver finished running successfully and there really is
only one pod?
It turns out that it will now recreate it (get or create).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
