ucaeon commented on code in PR #73591:
URL: https://github.com/apache/airflow/pull/73591#discussion_r4310742486
##########
dev/breeze/src/airflow_breeze/commands/kubernetes_kustomize_commands.py:
##########
@@ -279,6 +280,61 @@ def _wait_for_verify_resource(
return 1
+def _get_pod_health_problems(pods: list[dict[str, Any]]) -> list[str]:
+ problems = []
+ for pod in pods:
+ name = pod["metadata"]["name"]
+ status = pod.get("status", {})
+ phase = status.get("phase", "Unknown")
+ if pod["metadata"].get("deletionTimestamp"):
+ problems.append(f"{name}: terminating")
+ elif phase == "Succeeded":
+ continue
+ elif phase != "Running":
+ problems.append(f"{name}: {phase}")
+ elif not any(
+ condition["type"] == "Ready" and condition["status"] == "True"
+ for condition in status.get("conditions", [])
+ ):
+ problems.append(f"{name}: not Ready")
+ return problems
+
+
+def _wait_for_airflow_pods(
+ namespace: str, release_name: str, timeout_seconds: int, env: dict[str,
str]
+) -> int:
+ """Wait for the Helm release's pods to be Running/Ready or successfully
completed."""
+ kubectl = str(KUBECTL_BIN_PATH)
+ scope = ["-n", namespace, "-l", f"release={release_name}"]
+ console_print(f"[info]Checking Airflow pod health for release
{release_name} in {namespace}...")
+ deadline = time.monotonic() + timeout_seconds
+ problems = ["No healthy Airflow pods observed"]
+ while time.monotonic() < deadline:
+ result = run_command(
+ [kubectl, "get", "pods", *scope, "-o", "json",
"--request-timeout=10s"],
+ env=env,
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+ if result.returncode != 0:
+ console_print(f"[error]Could not check Airflow pod health:
{result.stderr}")
+ return 1
+ pods = json.loads(result.stdout)["items"]
+ problems = _get_pod_health_problems(pods)
Review Comment:
I followed your suggestions to retry failed queries and fail immediately on
terminal waiting reasons.
I also limited skipping the health check to actual dry-run mode so empty
output during a real run is retried instead of treated as success.
I updated the query failure test to cover retries and timeout. All 63 tests
and the full Kerberos smoke test on KinD passed. Thanks again for the helpful
feedback! ☺️
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
