This is an automated email from the ASF dual-hosted git repository. bbovenzi pushed a commit to branch autorefresh-w-basedate in repository https://gitbox.apache.org/repos/asf/airflow.git
commit cbc86a1bd210ebefeb12c759bad3b3e391244f6d Author: Brent Bovenzi <[email protected]> AuthorDate: Mon Nov 15 16:12:54 2021 -0600 update tree data fetching - include `base_date` in refresh api request - handle run order in webserver - add test for refresh to auto-stop --- airflow/www/jest-setup.js | 6 ++++- airflow/www/package.json | 1 + airflow/www/static/js/tree/useTreeData.js | 15 ++++++----- airflow/www/static/js/tree/useTreeData.test.js | 36 +++++++++++++++++--------- airflow/www/templates/airflow/tree.html | 1 + airflow/www/views.py | 3 ++- airflow/www/yarn.lock | 22 +++++++++++++++- 7 files changed, 62 insertions(+), 22 deletions(-) diff --git a/airflow/www/jest-setup.js b/airflow/www/jest-setup.js index cbab0b6..c8ff532 100644 --- a/airflow/www/jest-setup.js +++ b/airflow/www/jest-setup.js @@ -1,3 +1,5 @@ +// We need this lint rule for now because these are only dev-dependencies +/* eslint-disable import/no-extraneous-dependencies */ /*! * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -17,8 +19,10 @@ * under the License. */ -// eslint-disable-next-line import/no-extraneous-dependencies import '@testing-library/jest-dom'; +import { enableFetchMocks } from 'jest-fetch-mock'; + +enableFetchMocks(); // Mock a global object we use across the app global.stateColors = { diff --git a/airflow/www/package.json b/airflow/www/package.json index a77e2eb..c221982 100644 --- a/airflow/www/package.json +++ b/airflow/www/package.json @@ -55,6 +55,7 @@ "file-loader": "^6.0.0", "imports-loader": "^1.1.0", "jest": "^27.3.1", + "jest-fetch-mock": "^3.0.3", "mini-css-extract-plugin": "1.6.0", "moment": "^2.29.1", "moment-locales-webpack-plugin": "^1.2.0", diff --git a/airflow/www/static/js/tree/useTreeData.js b/airflow/www/static/js/tree/useTreeData.js index 7d7f8f0..4dcfaf2 100644 --- a/airflow/www/static/js/tree/useTreeData.js +++ b/airflow/www/static/js/tree/useTreeData.js @@ -31,6 +31,7 @@ const treeDataUrl = getMetaValue('tree_data'); const numRuns = getMetaValue('num_runs'); const urlRoot = getMetaValue('root'); const isPaused = getMetaValue('is_paused'); +const baseDate = getMetaValue('base_date'); const areActiveRuns = (runs) => runs.filter((run) => ['queued', 'running', 'scheduled'].includes(run.state)).length > 0; @@ -46,20 +47,19 @@ const formatData = (data) => { if (typeof data === 'string') formattedData = JSON.parse(data); // change from pacal to camelcase formattedData = camelcaseKeys(formattedData, { deep: true }); - // make sure dagRuns are sorted by date - formattedData.dagRuns = formattedData.dagRuns - .sort((a, b) => new Date(a.dataIntervalStart) - new Date(b.dataIntervalStart)); return formattedData; }; const useTreeData = () => { const [data, setData] = useState(formatData(treeData)); const defaultIsOpen = isPaused !== 'True' && !JSON.parse(localStorage.getItem('disableAutoRefresh')) && areActiveRuns(data.dagRuns); - const { isOpen: isRefreshOn, onToggle } = useDisclosure({ defaultIsOpen }); + const { isOpen: isRefreshOn, onToggle, onClose } = useDisclosure({ defaultIsOpen }); const handleRefresh = useCallback(async () => { try { - const resp = await fetch(`${treeDataUrl}?dag_id=${dagId}&num_runs=${numRuns}&root=${urlRoot}`); + const root = urlRoot ? `&root=${urlRoot}` : ''; + const base = baseDate ? `&base_date=${baseDate}` : ''; + const resp = await fetch(`${treeDataUrl}?dag_id=${dagId}&num_runs=${numRuns}${root}${base}`); let newData = await resp.json(); if (newData) { newData = formatData(newData); @@ -67,12 +67,13 @@ const useTreeData = () => { setData(newData); } // turn off auto refresh if there are no active runs - if (!areActiveRuns(newData.dagRuns)) onToggle(); + if (!areActiveRuns(newData.dagRuns)) onClose(); } } catch (e) { + onClose(); console.error(e); } - }, [data, onToggle]); + }, [data, onClose]); const onToggleRefresh = () => { if (isRefreshOn) { diff --git a/airflow/www/static/js/tree/useTreeData.test.js b/airflow/www/static/js/tree/useTreeData.test.js index 5c511ae..85d0e21 100644 --- a/airflow/www/static/js/tree/useTreeData.test.js +++ b/airflow/www/static/js/tree/useTreeData.test.js @@ -20,11 +20,11 @@ import { renderHook } from '@testing-library/react-hooks'; import useTreeData from './useTreeData'; -/* global describe, test, expect */ +/* global describe, test, expect, fetch, beforeEach */ global.autoRefreshInterval = 5; -const treeData = { +const pendingTreeData = { groups: {}, dag_runs: [ { @@ -41,9 +41,18 @@ const treeData = { ], }; +const finalTreeData = { + groups: {}, + dag_runs: [{ ...pendingTreeData.dag_runs[0], state: 'failed' }], +}; + describe('Test useTreeData hook', () => { + beforeEach(() => { + fetch.resetMocks(); + }); + test('data is valid camelcase json', () => { - global.treeData = JSON.stringify(treeData); + global.treeData = JSON.stringify(pendingTreeData); const { result } = renderHook(() => useTreeData()); const { data, isRefreshOn, onToggleRefresh } = result.current; @@ -55,20 +64,23 @@ describe('Test useTreeData hook', () => { expect(typeof onToggleRefresh).toBe('function'); }); - test('data with an unfinished state should have refresh on by default', () => { - global.treeData = JSON.stringify(treeData); + test('queued run should have refreshOn by default and then turn off when run failed', async () => { + // return a dag run of failed during refresh + fetch.mockResponse(JSON.stringify(finalTreeData)); + global.treeData = JSON.stringify(pendingTreeData); + global.autoRefreshInterval = 0.1; - const { result } = renderHook(() => useTreeData()); - const { isRefreshOn } = result.current; + const { result, waitFor } = renderHook(() => useTreeData()); - expect(isRefreshOn).toBe(true); + expect(result.current.isRefreshOn).toBe(true); + + await waitFor(() => expect(fetch).toBeCalled()); + + expect(result.current.isRefreshOn).toBe(false); }); test('data with a finished state should have refresh off by default', () => { - global.treeData = JSON.stringify({ - groups: {}, - dag_runs: [{ ...treeData.dag_runs[0], state: 'failed' }], - }); + global.treeData = JSON.stringify(finalTreeData); const { result } = renderHook(() => useTreeData()); const { isRefreshOn } = result.current; diff --git a/airflow/www/templates/airflow/tree.html b/airflow/www/templates/airflow/tree.html index 728b8f4..d76de75 100644 --- a/airflow/www/templates/airflow/tree.html +++ b/airflow/www/templates/airflow/tree.html @@ -25,6 +25,7 @@ {{ super() }} <meta name="num_runs" content="{{ num_runs }}"> <meta name="root" content="{{ root if root else '' }}"> + <meta name="base_date" content="{{ request.args.get('base_date') if request.args.get('base_date') else '' }}"> {% endblock %} {% block content %} diff --git a/airflow/www/views.py b/airflow/www/views.py index 5fe5966..c958944 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -288,7 +288,6 @@ def task_group_to_tree(task_item_or_group, dag, dag_runs, tis): } group_summaries = [get_summary(dr, children) for dr in dag_runs] - group_summaries.reverse() return { 'id': task_group.group_id, @@ -2272,6 +2271,7 @@ class Airflow(AirflowBaseView): .limit(num_runs) .all() ) + dag_runs.reverse() encoded_runs = [wwwutils.encode_dag_run(dr) for dr in dag_runs] dag_run_dates = {dr.execution_date: alchemy_to_dict(dr) for dr in dag_runs} @@ -3017,6 +3017,7 @@ class Airflow(AirflowBaseView): .limit(num_runs) .all() ) + dag_runs.reverse() encoded_runs = [wwwutils.encode_dag_run(dr) for dr in dag_runs] dag_run_dates = {dr.execution_date: alchemy_to_dict(dr) for dr in dag_runs} min_date = min(dag_run_dates, default=None) diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock index e0f6be1..8434813 100644 --- a/airflow/www/yarn.lock +++ b/airflow/www/yarn.lock @@ -4064,6 +4064,13 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7: safe-buffer "^5.0.1" sha.js "^2.4.8" +cross-fetch@^3.0.4: + version "3.1.4" + resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.4.tgz#9723f3a3a247bf8b89039f3a380a9244e8fa2f39"; + integrity sha512-1eAtFWdIubi6T4XPy6ei9iUFoKpUkIF971QLN8lIvvvwueI65+Nw5haMNKUwfJxabqlIIDODJKGrQ66gxC0PbQ== + dependencies: + node-fetch "2.6.1" + cross-spawn@^6.0.5: version "6.0.5" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4"; @@ -6923,6 +6930,14 @@ jest-environment-node@^27.3.1: jest-mock "^27.3.0" jest-util "^27.3.1" +jest-fetch-mock@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/jest-fetch-mock/-/jest-fetch-mock-3.0.3.tgz#31749c456ae27b8919d69824f1c2bd85fe0a1f3b"; + integrity sha512-Ux1nWprtLrdrH4XwE7O7InRY6psIi3GOsqNESJgMJ+M5cv4A8Lh7SN9d2V2kKRZ8ebAfcd1LNyZguAOb6JiDqw== + dependencies: + cross-fetch "^3.0.4" + promise-polyfill "^8.1.3" + jest-get-type@^27.3.1: version "27.3.1" resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.3.1.tgz#a8a2b0a12b50169773099eee60a0e6dd11423eff"; @@ -8064,7 +8079,7 @@ node-fetch-h2@^2.3.0: dependencies: http2-client "^1.2.5" -node-fetch@^2.6.1: [email protected], node-fetch@^2.6.1: version "2.6.1" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.1.tgz#045bd323631f76ed2e2b55573394416b639a0052"; integrity sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw== @@ -9115,6 +9130,11 @@ promise-inflight@^1.0.1: resolved "https://registry.yarnpkg.com/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3"; integrity sha1-mEcocL8igTL8vdhoEputEsPAKeM= +promise-polyfill@^8.1.3: + version "8.2.1" + resolved "https://registry.yarnpkg.com/promise-polyfill/-/promise-polyfill-8.2.1.tgz#1fa955b325bee4f6b8a4311e18148d4e5b46d254"; + integrity sha512-3p9zj0cEHbp7NVUxEYUWjQlffXqnXaZIMPkAO7HhFh8u5636xLRDHOUo2vpWSK0T2mqm6fKLXYn1KP6PAZ2gKg== + prompts@^2.0.1: version "2.4.2" resolved "https://registry.yarnpkg.com/prompts/-/prompts-2.4.2.tgz#7b57e73b3a48029ad10ebd44f74b01722a4cb069";
