This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 035553c869 Reduce grid view API calls (#24083)
035553c869 is described below
commit 035553c86988f403b43ef5825715b45f055d62dd
Author: Brent Bovenzi <[email protected]>
AuthorDate: Thu Jun 2 19:48:53 2022 +0200
Reduce grid view API calls (#24083)
* Reduce API calls from /grid
- Separate /grid_data from /grid
- Remove need for formatData
- Increase default query stale time to prevent extra fetches
- Fix useTask query keys
* consolidate grid data functions
* fix www tests
test grid_data instead of /grid
---
airflow/www/static/js/grid/Main.jsx | 26 +++++---
airflow/www/static/js/grid/api/useGridData.js | 7 +-
.../gridData.test.js => api/useGridData.test.js} | 2 +-
.../www/static/js/grid/api/useGridData.test.jsx | 78 ----------------------
airflow/www/static/js/grid/api/useTasks.js | 4 +-
airflow/www/static/js/grid/context/autorefresh.jsx | 15 +----
airflow/www/static/js/grid/dagRuns/index.test.jsx | 10 +--
airflow/www/static/js/grid/details/content/Dag.jsx | 3 +-
airflow/www/static/js/grid/index.jsx | 1 +
airflow/www/static/js/grid/utils/gridData.js | 34 ----------
airflow/www/templates/airflow/grid.html | 1 -
airflow/www/views.py | 11 ---
tests/www/views/test_views_acl.py | 10 +--
tests/www/views/test_views_tasks.py | 18 ++---
14 files changed, 38 insertions(+), 182 deletions(-)
diff --git a/airflow/www/static/js/grid/Main.jsx
b/airflow/www/static/js/grid/Main.jsx
index 98aa360aa6..5e9a4f2a9a 100644
--- a/airflow/www/static/js/grid/Main.jsx
+++ b/airflow/www/static/js/grid/Main.jsx
@@ -25,17 +25,21 @@ import {
Flex,
useDisclosure,
Divider,
+ Spinner,
} from '@chakra-ui/react';
+import { isEmpty } from 'lodash';
import Details from './details';
import useSelection from './utils/useSelection';
import Grid from './Grid';
import FilterBar from './FilterBar';
import LegendRow from './LegendRow';
+import { useGridData } from './api';
const detailsPanelKey = 'hideDetailsPanel';
const Main = () => {
+ const { data: { groups }, isLoading } = useGridData();
const isPanelOpen = localStorage.getItem(detailsPanelKey) !== 'true';
const { isOpen, onToggle } = useDisclosure({ defaultIsOpen: isPanelOpen });
const { clearSelection } = useSelection();
@@ -57,14 +61,20 @@ const Main = () => {
<LegendRow setHoveredTaskState={setHoveredTaskState} />
<Divider mb={5} borderBottomWidth={2} />
<Flex justifyContent="space-between">
- <Grid
- isPanelOpen={isOpen}
- onPanelToggle={onPanelToggle}
- hoveredTaskState={hoveredTaskState}
- />
- <Box borderLeftWidth={isOpen ? 1 : 0} position="relative">
- {isOpen && (<Details />)}
- </Box>
+ {isLoading || isEmpty(groups)
+ ? (<Spinner />)
+ : (
+ <>
+ <Grid
+ isPanelOpen={isOpen}
+ onPanelToggle={onPanelToggle}
+ hoveredTaskState={hoveredTaskState}
+ />
+ <Box borderLeftWidth={isOpen ? 1 : 0} position="relative">
+ {isOpen && (<Details />)}
+ </Box>
+ </>
+ )}
</Flex>
</Box>
);
diff --git a/airflow/www/static/js/grid/api/useGridData.js
b/airflow/www/static/js/grid/api/useGridData.js
index d31712989b..38d4e00748 100644
--- a/airflow/www/static/js/grid/api/useGridData.js
+++ b/airflow/www/static/js/grid/api/useGridData.js
@@ -17,14 +17,13 @@
* under the License.
*/
-/* global autoRefreshInterval, gridData */
+/* global autoRefreshInterval */
import { useQuery } from 'react-query';
import axios from 'axios';
import { getMetaValue } from '../../utils';
import { useAutoRefresh } from '../context/autorefresh';
-import { areActiveRuns, formatData } from '../utils/gridData';
import useErrorToast from '../utils/useErrorToast';
import useFilters, {
BASE_DATE_PARAM, NUM_RUNS_PARAM, RUN_STATE_PARAM, RUN_TYPE_PARAM, now,
@@ -42,8 +41,9 @@ const emptyData = {
groups: {},
};
+export const areActiveRuns = (runs = []) => runs.filter((run) => ['queued',
'running', 'scheduled'].includes(run.state)).length > 0;
+
const useGridData = () => {
- const initialData = formatData(gridData, emptyData);
const { isRefreshOn, stopRefresh } = useAutoRefresh();
const errorToast = useErrorToast();
const {
@@ -75,7 +75,6 @@ const useGridData = () => {
throw (error);
}
}, {
- initialData,
placeholderData: emptyData,
// only refetch if the refresh switch is on
refetchInterval: isRefreshOn && autoRefreshInterval * 1000,
diff --git a/airflow/www/static/js/grid/utils/gridData.test.js
b/airflow/www/static/js/grid/api/useGridData.test.js
similarity index 96%
rename from airflow/www/static/js/grid/utils/gridData.test.js
rename to airflow/www/static/js/grid/api/useGridData.test.js
index 6bbd8bf8b6..29a7f1ac8a 100644
--- a/airflow/www/static/js/grid/utils/gridData.test.js
+++ b/airflow/www/static/js/grid/api/useGridData.test.js
@@ -19,7 +19,7 @@
/* global describe, test, expect */
-import { areActiveRuns } from './gridData';
+import { areActiveRuns } from './useGridData';
describe('Test areActiveRuns()', () => {
test('Correctly detects active runs', () => {
diff --git a/airflow/www/static/js/grid/api/useGridData.test.jsx
b/airflow/www/static/js/grid/api/useGridData.test.jsx
deleted file mode 100644
index 24aece6f59..0000000000
--- a/airflow/www/static/js/grid/api/useGridData.test.jsx
+++ /dev/null
@@ -1,78 +0,0 @@
-/*!
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/* global describe, test, expect, beforeAll */
-
-import { renderHook } from '@testing-library/react-hooks';
-import useGridData from './useGridData';
-import { Wrapper } from '../utils/testUtils';
-
-const pendingGridData = {
- groups: {},
- dag_runs: [
- {
- dag_id: 'example_python_operator',
- run_id: 'manual__2021-11-08T21:14:17.170046+00:00',
- start_date: null,
- end_date: null,
- state: 'queued',
- execution_date: '2021-11-08T21:14:17.170046+00:00',
- data_interval_start: '2021-11-08T21:14:17.170046+00:00',
- data_interval_end: '2021-11-08T21:14:17.170046+00:00',
- run_type: 'manual',
- },
- ],
-};
-
-describe('Test useGridData hook', () => {
- beforeAll(() => {
- global.autoRefreshInterval = 5;
- });
-
- test('data is valid camelcase json', () => {
- global.gridData = JSON.stringify(pendingGridData);
-
- const { result } = renderHook(() => useGridData(), { wrapper: Wrapper });
- const { data } = result.current;
-
- expect(typeof data === 'object').toBe(true);
- expect(data.dagRuns).toBeDefined();
- expect(data.dag_runs).toBeUndefined();
- });
-
- test('Can handle no gridData', () => {
- global.gridData = null;
-
- const { result } = renderHook(() => useGridData(), { wrapper: Wrapper });
- const { data } = result.current;
-
- expect(data.dagRuns).toStrictEqual([]);
- expect(data.groups).toStrictEqual({});
- });
-
- test('Can handle empty gridData object', () => {
- global.gridData = {};
-
- const { result } = renderHook(() => useGridData(), { wrapper: Wrapper });
- const { data } = result.current;
-
- expect(data.dagRuns).toStrictEqual([]);
- expect(data.groups).toStrictEqual({});
- });
-});
diff --git a/airflow/www/static/js/grid/api/useTasks.js
b/airflow/www/static/js/grid/api/useTasks.js
index 9d0f56d7cb..a635622e27 100644
--- a/airflow/www/static/js/grid/api/useTasks.js
+++ b/airflow/www/static/js/grid/api/useTasks.js
@@ -23,9 +23,9 @@ import { getMetaValue } from '../../utils';
const tasksUrl = getMetaValue('tasks_api');
-export default function useTasks(dagId) {
+export default function useTasks() {
return useQuery(
- ['tasks', dagId],
+ 'tasks',
() => axios.get(tasksUrl),
{
placeholderData: { tasks: [] },
diff --git a/airflow/www/static/js/grid/context/autorefresh.jsx
b/airflow/www/static/js/grid/context/autorefresh.jsx
index ae242e5bf6..7fbdc405f9 100644
--- a/airflow/www/static/js/grid/context/autorefresh.jsx
+++ b/airflow/www/static/js/grid/context/autorefresh.jsx
@@ -17,11 +17,10 @@
* under the License.
*/
-/* global localStorage, gridData, document */
+/* global localStorage, document */
import React, { useContext, useState, useEffect } from 'react';
import { getMetaValue } from '../../utils';
-import { formatData, areActiveRuns } from '../utils/gridData';
const autoRefreshKey = 'disabledAutoRefresh';
@@ -31,17 +30,9 @@ const isRefreshDisabled =
JSON.parse(localStorage.getItem(autoRefreshKey));
const AutoRefreshContext = React.createContext(null);
export const AutoRefreshProvider = ({ children }) => {
- let dagRuns = [];
- try {
- const data = JSON.parse(gridData);
- if (data.dag_runs) dagRuns = formatData(data.dag_runs);
- } catch {
- dagRuns = [];
- }
const [isPaused, setIsPaused] = useState(initialIsPaused);
- const isActive = areActiveRuns(dagRuns);
const isRefreshAllowed = !(isPaused || isRefreshDisabled);
- const initialState = isRefreshAllowed && isActive;
+ const initialState = isRefreshAllowed;
const [isRefreshOn, setRefresh] = useState(initialState);
@@ -67,8 +58,6 @@ export const AutoRefreshProvider = ({ children }) => {
setIsPaused(!e.value);
if (!e.value) {
stopRefresh();
- } else if (isActive) {
- setRefresh(true);
}
};
diff --git a/airflow/www/static/js/grid/dagRuns/index.test.jsx
b/airflow/www/static/js/grid/dagRuns/index.test.jsx
index a507df522c..0420d7aba8 100644
--- a/airflow/www/static/js/grid/dagRuns/index.test.jsx
+++ b/airflow/www/static/js/grid/dagRuns/index.test.jsx
@@ -112,15 +112,7 @@ describe('Test DagRuns', () => {
});
test('Handles empty data correctly', () => {
- global.gridData = null;
- const { queryByTestId } = render(
- <DagRuns />, { wrapper: TableWrapper },
- );
- expect(queryByTestId('run')).toBeNull();
- });
-
- test('Handles no data correctly', () => {
- global.gridData = {};
+ global.autoRefreshInterval = 0;
const { queryByTestId } = render(
<DagRuns />, { wrapper: TableWrapper },
);
diff --git a/airflow/www/static/js/grid/details/content/Dag.jsx
b/airflow/www/static/js/grid/details/content/Dag.jsx
index 0f434d88fd..e527b494cf 100644
--- a/airflow/www/static/js/grid/details/content/Dag.jsx
+++ b/airflow/www/static/js/grid/details/content/Dag.jsx
@@ -37,11 +37,10 @@ import { useTasks, useGridData } from '../../api';
import Time from '../../components/Time';
import { SimpleStatus } from '../../components/StatusBox';
-const dagId = getMetaValue('dag_id');
const dagDetailsUrl = getMetaValue('dag_details_url');
const Dag = () => {
- const { data: taskData } = useTasks(dagId);
+ const { data: taskData } = useTasks();
const { data: { dagRuns } } = useGridData();
if (!taskData) return null;
const { tasks = [], totalEntries = '' } = taskData;
diff --git a/airflow/www/static/js/grid/index.jsx
b/airflow/www/static/js/grid/index.jsx
index 3d8702841e..b8c6db5324 100644
--- a/airflow/www/static/js/grid/index.jsx
+++ b/airflow/www/static/js/grid/index.jsx
@@ -48,6 +48,7 @@ const queryClient = new QueryClient({
refetchOnWindowFocus: false,
retry: 1,
retryDelay: 500,
+ staleTime: 5 * 60 * 1000, // 5 minutes
refetchOnMount: true, // Refetches stale queries, not "always"
},
mutations: {
diff --git a/airflow/www/static/js/grid/utils/gridData.js
b/airflow/www/static/js/grid/utils/gridData.js
deleted file mode 100644
index 95171652b7..0000000000
--- a/airflow/www/static/js/grid/utils/gridData.js
+++ /dev/null
@@ -1,34 +0,0 @@
-/*!
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import camelcaseKeys from 'camelcase-keys';
-
-export const areActiveRuns = (runs = []) => runs.filter((run) => ['queued',
'running', 'scheduled'].includes(run.state)).length > 0;
-
-export const formatData = (data, emptyData) => {
- if (!data || !Object.keys(data).length) {
- return emptyData;
- }
- let formattedData = data;
- // Convert to json if needed
- if (typeof data === 'string') formattedData = JSON.parse(data);
- // change from pascal to camelcase
- formattedData = camelcaseKeys(formattedData, { deep: true });
- return formattedData;
-};
diff --git a/airflow/www/templates/airflow/grid.html
b/airflow/www/templates/airflow/grid.html
index 46d52f7d27..f72e0ed4f1 100644
--- a/airflow/www/templates/airflow/grid.html
+++ b/airflow/www/templates/airflow/grid.html
@@ -39,7 +39,6 @@
{% block tail_js %}
{{ super()}}
<script>
- const gridData = {{ data|tojson }};
const stateColors = {{ state_color_mapping|tojson }};
const autoRefreshInterval = {{ auto_refresh_interval }};
const defaultDagRunDisplayNumber = {{ default_dag_run_display_number }};
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 04b955089f..25b60cdac9 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2611,8 +2611,6 @@ class Airflow(AirflowBaseView):
.limit(num_runs)
.all()
)
- dag_runs.reverse()
- encoded_runs = [wwwutils.encode_dag_run(dr) for dr in dag_runs]
dag_run_dates = {dr.execution_date: alchemy_to_dict(dr) for dr in
dag_runs}
max_date = max(dag_run_dates, default=None)
@@ -2632,14 +2630,6 @@ class Airflow(AirflowBaseView):
else:
external_log_name = None
- data = {
- 'groups': task_group_to_grid(dag.task_group, dag, dag_runs,
session),
- 'dag_runs': encoded_runs,
- }
-
- # avoid spaces to reduce payload size
- data = htmlsafe_json_dumps(data, separators=(',', ':'))
-
default_dag_run_display_number = conf.getint('webserver',
'default_dag_run_display_number')
num_runs_options = [5, 25, 50, 100, 365]
@@ -2654,7 +2644,6 @@ class Airflow(AirflowBaseView):
form=form,
dag=dag,
doc_md=doc_md,
- data=data,
num_runs=num_runs,
show_external_log_redirect=task_log_reader.supports_external_link,
external_log_name=external_log_name,
diff --git a/tests/www/views/test_views_acl.py
b/tests/www/views/test_views_acl.py
index d96098a5c6..84003bf366 100644
--- a/tests/www/views/test_views_acl.py
+++ b/tests/www/views/test_views_acl.py
@@ -563,7 +563,7 @@ DURATION_URL =
"duration?days=30&dag_id=example_bash_operator"
TRIES_URL = "tries?days=30&dag_id=example_bash_operator"
LANDING_TIMES_URL = "landing_times?days=30&dag_id=example_bash_operator"
GANTT_URL = "gantt?dag_id=example_bash_operator"
-TREE_URL = "tree?dag_id=example_bash_operator"
+GRID_DATA_URL = "object/grid_data?dag_id=example_bash_operator"
LOG_URL = (
f"log?task_id=runme_0&dag_id=example_bash_operator&"
f"execution_date={urllib.parse.quote_plus(str(DEFAULT_DATE))}"
@@ -581,8 +581,8 @@ LOG_URL = (
("client_all_dags_tis", TRIES_URL, "example_bash_operator"),
("client_all_dags_tis", LANDING_TIMES_URL, "example_bash_operator"),
("client_all_dags_tis", GANTT_URL, "example_bash_operator"),
- ("client_dags_tis_logs", TREE_URL, "runme_1"),
- ("viewer_client", TREE_URL, "runme_1"),
+ ("client_dags_tis_logs", GRID_DATA_URL, "runme_1"),
+ ("viewer_client", GRID_DATA_URL, "runme_1"),
("client_dags_tis_logs", LOG_URL, "Log by attempts"),
("user_client", LOG_URL, "Log by attempts"),
],
@@ -595,8 +595,8 @@ LOG_URL = (
"tries",
"landing-times",
"gantt",
- "tree-for-readonly-role",
- "tree-for-viewer",
+ "grid-data-for-readonly-role",
+ "grid-data-for-viewer",
"log",
"log-for-user",
],
diff --git a/tests/www/views/test_views_tasks.py
b/tests/www/views/test_views_tasks.py
index 3428ac848a..d99a8fd5ca 100644
--- a/tests/www/views/test_views_tasks.py
+++ b/tests/www/views/test_views_tasks.py
@@ -165,24 +165,14 @@ def client_ti_without_dag_edit(app):
id='graph',
),
pytest.param(
- 'tree?dag_id=example_bash_operator',
+ 'object/grid_data?dag_id=example_bash_operator',
['runme_1'],
- id='tree',
+ id='grid-data',
),
pytest.param(
- 'dags/example_bash_operator/grid',
- ['runme_1'],
- id='grid',
- ),
- pytest.param(
- 'tree?dag_id=example_subdag_operator.section-1',
- ['section-1-task-1'],
- id="tree-subdag-url-param",
- ),
- pytest.param(
- 'dags/example_subdag_operator.section-1/grid',
+ 'object/grid_data?dag_id=example_subdag_operator.section-1',
['section-1-task-1'],
- id="grid-subdag",
+ id="grid-data-subdag",
),
pytest.param(
'duration?days=30&dag_id=example_bash_operator',
