xvega commented on code in PR #62369:
URL: https://github.com/apache/airflow/pull/62369#discussion_r2927424256
##########
airflow-core/src/airflow/ui/src/queries/useGridTISummaries.ts:
##########
@@ -16,45 +16,110 @@
* specific language governing permissions and limitations
* under the License.
*/
-import { useGridServiceGetGridTiSummaries } from "openapi/queries";
-import type { TaskInstanceState } from "openapi/requests";
+import { useEffect, useState } from "react";
+
+import { OpenAPI } from "openapi/requests/core/OpenAPI";
+import type { GridTISummaries, TaskInstanceState } from "openapi/requests";
import { isStatePending, useAutoRefresh } from "src/utils";
-export const useGridTiSummaries = ({
+/**
+ * Streams TI summaries for all grid runs over a single HTTP connection
(NDJSON).
+ *
+ * The server emits one JSON line per Dag run as soon as that run's task
+ * instances have been computed, so the grid renders each column progressively
+ * rather than waiting for the entire payload. This eliminates the N+1 request
+ * pattern without loading all runs into one large query.
+ *
+ * Auto-refreshes while any run is still in a pending state.
+ */
+export const useGridTiSummariesStream = ({
dagId,
- enabled,
- isSelected,
- runId,
- state,
+ runIds,
+ states,
}: {
dagId: string;
- enabled?: boolean;
- isSelected?: boolean;
- runId: string;
- state?: TaskInstanceState | null | undefined;
+ runIds: Array<string>;
+ states?: Array<TaskInstanceState | null | undefined>;
}) => {
+ const [summariesByRunId, setSummariesByRunId] = useState<Map<string,
GridTISummaries>>(new Map());
+ const [isStreaming, setIsStreaming] = useState(false);
+ const [refreshTick, setRefreshTick] = useState(0);
+
const baseRefetchInterval = useAutoRefresh({ dagId });
- const slowRefreshMultiplier = 5;
- const refetchInterval =
- typeof baseRefetchInterval === "number"
- ? baseRefetchInterval * (isSelected ? 1 : slowRefreshMultiplier)
- : baseRefetchInterval;
-
- const { data: gridTiSummaries, ...rest } = useGridServiceGetGridTiSummaries(
- {
- dagId,
- runId,
- },
- undefined,
- {
- enabled: Boolean(runId) && Boolean(dagId) && enabled,
- placeholderData: (prev) => prev,
- refetchInterval: (query) =>
- ((state !== undefined && isStatePending(state)) ||
- query.state.data?.task_instances.some((ti) =>
isStatePending(ti.state))) &&
- refetchInterval,
- },
- );
-
- return { data: gridTiSummaries, ...rest };
+ const hasActiveRuns = states?.some((s) => s !== undefined &&
isStatePending(s)) ?? false;
+
+ // Stable key so the effect only re-fires when the run list actually changes.
+ const runIdsKey = runIds.join(",");
+
+ // Stream (or re-stream) whenever the run list or refresh tick changes.
+ useEffect(() => {
+ if (!dagId || runIds.length === 0) return;
+
+ const abortController = new AbortController();
+ let reader: ReadableStreamDefaultReader<Uint8Array> | null = null;
+
+ const fetchStream = async () => {
+ setIsStreaming(true);
+ // Keep stale data visible while the new stream loads — columns update in
+ // place as fresh lines arrive rather than flashing blank.
+ try {
+ const queryString = runIds.map((id) =>
`run_ids=${encodeURIComponent(id)}`).join("&");
+ const response = await
fetch(`${OpenAPI.BASE}/ui/grid/ti_summaries/${dagId}?${queryString}`, {
+ signal: abortController.signal,
+ });
+
+ if (!response.ok || !response.body) return;
+
+ reader = response.body.getReader();
+ const decoder = new TextDecoder();
+ let buffer = "";
+
+ while (true) {
+ const { done, value } = await reader.read();
+
+ if (done) break;
+ buffer += decoder.decode(value, { stream: true });
+
+ // Each complete line is one serialised GridTISummaries object.
+ const lines = buffer.split("\n");
+ buffer = lines.pop() ?? "";
+
+ for (const line of lines) {
+ if (line.trim()) {
+ const summary = JSON.parse(line) as GridTISummaries;
+
+ setSummariesByRunId((prev) => new Map(prev).set(summary.run_id,
summary));
+ }
+ }
+ }
+ } catch (error) {
+ if ((error as Error).name !== "AbortError") {
+ console.error("TI summaries stream error:", error);
+ }
+ } finally {
+ setIsStreaming(false);
+ }
+ };
+
+ fetchStream();
+
+ return () => {
+ abortController.abort();
+ reader?.cancel();
+ };
+ // eslint-disable-next-line react-hooks/exhaustive-deps
Review Comment:
This is intentional, `runIds` is an array so its reference changes on every
render. `runIdsKey` is just a stable string version of it that only changes
when the actual content changes, preventing unnecessary re-streams.
The disable comment suppresses the lint warning about not listing runIds
directly.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
