kaxil commented on code in PR #76833: URL: https://github.com/apache/airflow/pull/76833#discussion_r4622372048
########## airflow-core/src/airflow/api_fastapi/core_api/routes/ui/durable_execution.py: ########## @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json + +from fastapi import Depends +from sqlalchemy import select + +from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity +from airflow.api_fastapi.common.db.common import SessionDep +from airflow.api_fastapi.common.router import AirflowRouter +from airflow.api_fastapi.core_api.datamodels.ui.durable_execution import DurableExecutionResponse +from airflow.api_fastapi.core_api.security import requires_access_dag +from airflow.api_fastapi.core_api.services.public.task_coordinates import TaskScopeDep +from airflow.models.task_state_store import TaskStateStoreModel + +# Must match DURABLE_RECORD_KEY in the task SDK, which writes the record. +DURABLE_RECORD_KEY = "__airflow_durable__" + +durable_execution_router = AirflowRouter( + tags=["Durable Execution"], + prefix="/dags/{dag_id}/dagRuns/{dag_run_id}/taskInstances/{task_id}/durable", +) + + +@durable_execution_router.get( + "", + dependencies=[Depends(requires_access_dag(method="GET", access_entity=DagAccessEntity.TASK_INSTANCE))], +) +def get_durable_execution(scope: TaskScopeDep, session: SessionDep) -> DurableExecutionResponse: + """Get the durable execution history of a task instance; empty when the task is not durable.""" + value = session.scalar( + select(TaskStateStoreModel.value).where( + TaskStateStoreModel.dag_id == scope.dag_id, + TaskStateStoreModel.run_id == scope.run_id, + TaskStateStoreModel.task_id == scope.task_id, + TaskStateStoreModel.region_id == scope.region_id, + TaskStateStoreModel.region_index == scope.region_index, + TaskStateStoreModel.key == DURABLE_RECORD_KEY, + ) + ) + if value is None: + return DurableExecutionResponse(tries=[]) + return DurableExecutionResponse.model_validate(json.loads(value)) Review Comment: With `[workers] state_store_backend` set (3.3.0+), `set()` offloads every value, so this row is only `{"__airflow_state_ref__": "<ref>"}`. `model_validate` raises on that and the endpoint 500s for every durable task on those deployments. Can we return `tries=[]` for a ref envelope or a record that does not validate, and add a test with the envelope? ########## task-sdk/src/airflow/sdk/bases/resumablejobmixin.py: ########## @@ -132,12 +146,27 @@ def execute_resumable(self, context: Context) -> Any: decision = self.get_resume_decision(context) if decision.action == "reconnect": - return self.poll_until_complete(decision.external_id, context) + self._record_try(context, DurableDecision.RECONNECTED, decision.external_id) + result = self.poll_until_complete(decision.external_id, context) + self._record_try(context, DurableDecision.RECONNECTED, decision.external_id, job_finished=True) Review Comment: This assumes `poll_until_complete` only returns once the job succeeded. When it raises (the documented "Raise on failure"), the try stays `status: "running"` forever, so a failed retry's banner says it is still waiting on a running job. When it returns early (Glue with `wait_for_completion=False`, Databricks with `wait_for_termination=False`), this writes `success` and a saved time while the job is still running. PowerBI's poll defers, so this line never runs for it. Could the poll be wrapped to record a terminal status on `except Exception` before re-raising, and only write `success` when `is_job_succeeded(get_job_status(...))` says so? ########## airflow-core/src/airflow/ui/src/pages/TaskInstance/Logs/DurableBanner.tsx: ########## @@ -0,0 +1,117 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Box, HStack, Text, VStack } from "@chakra-ui/react"; +import { useTranslation } from "react-i18next"; + +import { useDurableExecutionServiceGetDurableExecution } from "openapi/queries"; +import type { TaskInstanceResponse } from "openapi/requests"; + +import { Alert, Tooltip } from "src/system-components"; + +import { StateBadge } from "src/components/StateBadge"; +import Time from "src/components/Time"; + +import { useTaskInstanceCoordinates } from "src/hooks/useTaskInstanceCoordinates"; +import { isStatePending, useAutoRefresh } from "src/utils"; +import { renderDuration } from "src/utils/datetimeUtils"; + +export const DurableBanner = ({ + taskInstance, + tryNumber, +}: { + readonly taskInstance?: Pick< + TaskInstanceResponse, + "dag_id" | "dag_run_id" | "map_index" | "state" | "task_id" + >; + readonly tryNumber?: number; +}) => { + const { t: translate } = useTranslation(["dag", "common"]); + const coordinates = useTaskInstanceCoordinates(); + const refetchInterval = useAutoRefresh({ dagId: taskInstance?.dag_id }); + + const { data: record } = useDurableExecutionServiceGetDurableExecution( + { + ...coordinates, + dagId: taskInstance?.dag_id ?? "", + dagRunId: taskInstance?.dag_run_id ?? "", + mapIndex: taskInstance?.map_index ?? -1, + taskId: taskInstance?.task_id ?? "", + }, + undefined, + { + // A first try never reconnects or reuses a result, so only retries can show the banner. + enabled: taskInstance !== undefined && (tryNumber ?? 0) > 1, + refetchInterval: isStatePending(taskInstance?.state) && refetchInterval, Review Comment: The SDK writes the finished record just before the TI moves to success, so the last poll here usually still sees `running`, and once the state stops being pending this stops refetching. With the 5 minute `staleTime` the banner stays on "Waiting on the running job" until a reload, and the saved time is the number this banner exists to show. `useLogs` handles the same race with `dayjs(query.state.dataUpdatedAt).isBefore(taskInstance?.end_date)`; could this do the same? ########## airflow-core/src/airflow/ui/src/pages/TaskInstance/Logs/DurableBanner.tsx: ########## @@ -0,0 +1,117 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Box, HStack, Text, VStack } from "@chakra-ui/react"; +import { useTranslation } from "react-i18next"; + +import { useDurableExecutionServiceGetDurableExecution } from "openapi/queries"; +import type { TaskInstanceResponse } from "openapi/requests"; + +import { Alert, Tooltip } from "src/system-components"; + +import { StateBadge } from "src/components/StateBadge"; +import Time from "src/components/Time"; + +import { useTaskInstanceCoordinates } from "src/hooks/useTaskInstanceCoordinates"; +import { isStatePending, useAutoRefresh } from "src/utils"; +import { renderDuration } from "src/utils/datetimeUtils"; + +export const DurableBanner = ({ + taskInstance, + tryNumber, +}: { + readonly taskInstance?: Pick< + TaskInstanceResponse, + "dag_id" | "dag_run_id" | "map_index" | "state" | "task_id" + >; + readonly tryNumber?: number; +}) => { + const { t: translate } = useTranslation(["dag", "common"]); + const coordinates = useTaskInstanceCoordinates(); + const refetchInterval = useAutoRefresh({ dagId: taskInstance?.dag_id }); + + const { data: record } = useDurableExecutionServiceGetDurableExecution( + { + ...coordinates, + dagId: taskInstance?.dag_id ?? "", + dagRunId: taskInstance?.dag_run_id ?? "", + mapIndex: taskInstance?.map_index ?? -1, + taskId: taskInstance?.task_id ?? "", + }, + undefined, + { + // A first try never reconnects or reuses a result, so only retries can show the banner. + enabled: taskInstance !== undefined && (tryNumber ?? 0) > 1, + refetchInterval: isStatePending(taskInstance?.state) && refetchInterval, + }, + ); + + const entry = (record?.tries ?? []).find( + (attempt) => + attempt.try_number === tryNumber && + (attempt.decision === "reconnected" || attempt.decision === "already_succeeded"), + ); + + if (entry === undefined) { + return undefined; + } + + const seconds = entry.saved_seconds; + const externalId = + typeof entry.external_id === "string" ? entry.external_id : JSON.stringify(entry.external_id); + let detail = translate(`logs.durable.detail.${entry.decision}`); + + if (entry.decision === "reconnected") { + detail = + seconds === null Review Comment: This picks the waiting text from `saved_seconds === null` rather than the status. A finished reconnect with no `submitted_at` (try 1 ran an older SDK, or its record write failed) shows "Job: Success" next to "Waiting on the running job". Should this check `entry.status === "running"` instead? ########## task-sdk/src/airflow/sdk/bases/resumablejobmixin.py: ########## @@ -132,12 +146,27 @@ def execute_resumable(self, context: Context) -> Any: decision = self.get_resume_decision(context) if decision.action == "reconnect": - return self.poll_until_complete(decision.external_id, context) + self._record_try(context, DurableDecision.RECONNECTED, decision.external_id) + result = self.poll_until_complete(decision.external_id, context) + self._record_try(context, DurableDecision.RECONNECTED, decision.external_id, job_finished=True) + return result if decision.action == "already_succeeded": + self._record_try(context, DurableDecision.ALREADY_SUCCEEDED, decision.external_id) return self.get_job_result(decision.external_id, context) + # A "submit" decision covers both a first submit and one after a failed job; only the + # latter still has an id stored, which the banner shows as the job that was replaced. + task_state_store = context.get("task_state_store") + failed_id = task_state_store.get(self.external_id_key) if task_state_store is not None else None Review Comment: `get_resume_decision` already read this key and drops the ID on the terminal path, so this is a second store round trip on every submit. Returning it on the `ResumeDecision("submit", ...)` would avoid that. The comment above also says the banner shows the replaced job, but the banner ignores `resubmitted`. ########## task-sdk/src/airflow/sdk/bases/resumablejobmixin.py: ########## @@ -225,6 +254,51 @@ def persist_job_id(self, context: Context, external_id: JsonValue) -> None: external_id=external_id, ) + def _record_try( + self, + context: Context, + decision: DurableDecision, + external_id: JsonValue, + *, + job_finished: bool = False, + previous_external_id: JsonValue = None, + ) -> None: + """Record this try for the logs banner; a finished reconnect stores the job's run time as saved.""" + task_state_store = context.get("task_state_store") + if task_state_store is None: + return + try: + now = datetime.now(tz=UTC) + stored = task_state_store.get(DURABLE_RECORD_KEY) + record: dict[str, Any] = stored if isinstance(stored, dict) else {} + if decision in (DurableDecision.SUBMITTED, DurableDecision.RESUBMITTED): + record["submitted_at"] = now.isoformat() Review Comment: `submitted_at` is one field for the whole record, so a resubmit overwrites it. If try 1 submits A, try 2 reconnects to A and A fails, and try 3 submits B, try 2's tooltip then shows A's ID next to B's submit time. Should it live on each try entry? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
