This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch v2-7-test in repository https://gitbox.apache.org/repos/asf/airflow.git
commit dceeb134b47f55ed8c92806a722da1a0aeddc57a Author: Brent Bovenzi <[email protected]> AuthorDate: Thu Aug 3 07:15:27 2023 +0800 Remove old graph (#32958) (cherry picked from commit 154deed02ecc0be36b1ddfc9144aa05fb77b5f2d) --- airflow/www/forms.py | 6 - airflow/www/static/js/callModal.js | 391 ---------- .../static/js/cluster-activity/nav/FilterBar.tsx | 2 + airflow/www/static/js/components/Time.test.tsx | 11 +- airflow/www/static/js/components/Time.tsx | 6 +- airflow/www/static/js/dag.js | 51 ++ .../js/dag/details/taskInstance/Logs/utils.ts | 1 + .../www/static/js/dag/grid/dagRuns/index.test.tsx | 7 +- airflow/www/static/js/dag/nav/FilterBar.tsx | 1 + airflow/www/static/js/graph.js | 828 --------------------- airflow/www/static/js/task_instances.js | 49 +- airflow/www/templates/airflow/dag.html | 6 +- airflow/www/templates/airflow/graph.html | 146 ---- airflow/www/views.py | 101 +-- airflow/www/webpack.config.js | 2 +- docs/apache-airflow/img/graph.png | Bin 128870 -> 429818 bytes docs/apache-airflow/img/mapping-simple-graph.png | Bin 7676 -> 40118 bytes docs/apache-airflow/img/task_group.gif | Bin 609981 -> 137189 bytes tests/www/views/test_views_decorators.py | 12 +- tests/www/views/test_views_graph.py | 348 --------- tests/www/views/test_views_tasks.py | 35 +- 21 files changed, 119 insertions(+), 1884 deletions(-) diff --git a/airflow/www/forms.py b/airflow/www/forms.py index 9e143ce7b9..4a0213945c 100644 --- a/airflow/www/forms.py +++ b/airflow/www/forms.py @@ -121,12 +121,6 @@ class DateTimeWithNumRunsForm(FlaskForm): ) -class DateTimeWithNumRunsWithDagRunsForm(DateTimeWithNumRunsForm): - """Date time and number of runs and dag runs form for graph and gantt view.""" - - execution_date = SelectField("DAG run") - - class DagRunEditForm(DynamicForm): """Form for editing DAG Run. diff --git a/airflow/www/static/js/callModal.js b/airflow/www/static/js/callModal.js deleted file mode 100644 index 050c7eebff..0000000000 --- a/airflow/www/static/js/callModal.js +++ /dev/null @@ -1,391 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* global document, window, $ */ - -import { getMetaValue } from "./utils"; -import { formatDateTime } from "./datetime_utils"; - -function updateQueryStringParameter(uri, key, value) { - const re = new RegExp(`([?&])${key}=.*?(&|$)`, "i"); - const separator = uri.indexOf("?") !== -1 ? "&" : "?"; - if (uri.match(re)) { - return uri.replace(re, `$1${key}=${value}$2`); - } - - return `${uri}${separator}${key}=${value}`; -} - -function updateUriToFilterTasks(uri, taskId, filterUpstream, filterDownstream) { - const uriWithRoot = updateQueryStringParameter(uri, "root", taskId); - const uriWithFilterUpstreamQuery = updateQueryStringParameter( - uriWithRoot, - "filter_upstream", - filterUpstream - ); - return updateQueryStringParameter( - uriWithFilterUpstreamQuery, - "filter_downstream", - filterDownstream - ); -} - -const dagId = getMetaValue("dag_id"); -const logsWithMetadataUrl = getMetaValue("logs_with_metadata_url"); -const externalLogUrl = getMetaValue("external_log_url"); -const extraLinksUrl = getMetaValue("extra_links_url"); -const showExternalLogRedirect = - getMetaValue("show_external_log_redirect") === "True"; - -const buttons = Array.from( - document.querySelectorAll('a[id^="btn_"][data-base-url]') -).reduce((obj, elm) => { - // eslint-disable-next-line no-param-reassign - obj[elm.id.replace("btn_", "")] = elm; - return obj; -}, {}); - -function updateButtonUrl(elm, params) { - let url = elm.dataset.baseUrl; - if (params.dag_id && elm.dataset.baseUrl.indexOf(dagId) !== -1) { - url = url.replace(dagId, params.dag_id); - // eslint-disable-next-line no-param-reassign - delete params.dag_id; - } - if ( - Object.prototype.hasOwnProperty.call(params, "map_index") && - params.map_index === undefined - ) { - // eslint-disable-next-line no-param-reassign - delete params.map_index; - } - elm.setAttribute("href", `${url}?${$.param(params)}`); -} - -function updateModalUrls({ - executionDate, - subDagId, - taskId, - mapIndex, - dagRunId, -}) { - updateButtonUrl(buttons.subdag, { - dag_id: subDagId, - execution_date: executionDate, - }); - - updateButtonUrl(buttons.task, { - dag_id: dagId, - task_id: taskId, - execution_date: executionDate, - map_index: mapIndex, - }); - - updateButtonUrl(buttons.rendered, { - dag_id: dagId, - task_id: taskId, - execution_date: executionDate, - map_index: mapIndex, - }); - - updateButtonUrl(buttons.mapped, { - _flt_3_dag_id: dagId, - _flt_3_task_id: taskId, - _flt_3_run_id: dagRunId, - _oc_TaskInstanceModelView: "map_index", - }); - - if (buttons.rendered_k8s) { - updateButtonUrl(buttons.rendered_k8s, { - dag_id: dagId, - task_id: taskId, - execution_date: executionDate, - map_index: mapIndex, - }); - } - - const tiButtonParams = { - _flt_3_dag_id: dagId, - _flt_3_task_id: taskId, - _oc_TaskInstanceModelView: "dag_run.execution_date", - }; - // eslint-disable-next-line no-underscore-dangle - if (mapIndex >= 0) tiButtonParams._flt_0_map_index = mapIndex; - updateButtonUrl(buttons.ti, tiButtonParams); - - updateButtonUrl(buttons.log, { - dag_id: dagId, - task_id: taskId, - execution_date: executionDate, - map_index: mapIndex, - }); - - updateButtonUrl(buttons.xcom, { - dag_id: dagId, - task_id: taskId, - execution_date: executionDate, - map_index: mapIndex, - }); -} - -function callModal({ - taskId, - executionDate, - extraLinks, - tryNumber, - isSubDag, - dagRunId, - mapIndex = -1, - isMapped = false, - mappedStates = [], -}) { - // Turn off previous event listeners - $(".map_index_item").off("click"); - $("form[data-action]").off("submit"); - - const location = String(window.location); - $("#btn_filter_upstream").on("click", () => { - window.location = updateUriToFilterTasks(location, taskId, "true", "false"); - }); - $("#btn_filter_downstream").on("click", () => { - window.location = updateUriToFilterTasks(location, taskId, "false", "true"); - }); - $("#btn_filter_upstream_downstream").on("click", () => { - window.location = updateUriToFilterTasks(location, taskId, "true", "true"); - }); - $("#dag_run_id").text(dagRunId); - $("#task_id").text(taskId); - $("#execution_date").text(formatDateTime(executionDate)); - $("#taskInstanceModal").modal({}); - $("#taskInstanceModal").css("margin-top", "0"); - $("#extra_links").prev("hr").hide(); - $("#extra_links").empty().hide(); - if (mapIndex >= 0) { - $("#modal_map_index").show(); - $("#modal_map_index .value").text(mapIndex); - } else { - $("#modal_map_index").hide(); - $("#modal_map_index .value").text(""); - } - - let subDagId; - if (isSubDag) { - $("#div_btn_subdag").show(); - subDagId = `${dagId}.${taskId}`; - } else { - $("#div_btn_subdag").hide(); - } - - // Show a span or dropdown for mapIndex - if (mapIndex >= 0 && !mappedStates.length) { - $("#modal_map_index").show(); - $("#modal_map_index .value").text(mapIndex); - $("#mapped_dropdown").hide(); - } else if (mapIndex >= 0 || isMapped) { - $("#modal_map_index").show(); - $("#modal_map_index .value").text(""); - $("#mapped_dropdown").show(); - - const dropdownText = - mapIndex > -1 ? mapIndex : `All ${mappedStates.length} Mapped Instances`; - $("#mapped_dropdown #dropdown-label").text(dropdownText); - $("#mapped_dropdown .dropdown-menu").empty(); - $("#mapped_dropdown .dropdown-menu").append( - `<li><a href="#" class="map_index_item" data-mapIndex="all">All ${mappedStates.length} Mapped Instances</a></li>` - ); - mappedStates.forEach((state, i) => { - $("#mapped_dropdown .dropdown-menu").append( - `<li><a href="#" class="map_index_item" data-mapIndex="${i}">${i} - ${state}</a></li>` - ); - }); - } else { - $("#modal_map_index").hide(); - $("#modal_map_index .value").text(""); - $("#mapped_dropdown").hide(); - } - - if (isMapped) { - $("#task_actions").text( - `Task Actions for all ${mappedStates.length} instances` - ); - $("#btn_mapped").show(); - $("#mapped_dropdown").css("display", "inline-block"); - $("#btn_rendered").hide(); - $("#btn_xcom").hide(); - $("#btn_log").hide(); - $("#btn_task").hide(); - } else { - $("#task_actions").text("Task Actions"); - $("#btn_rendered").show(); - $("#btn_xcom").show(); - $("#btn_log").show(); - $("#btn_mapped").hide(); - $("#btn_task").show(); - } - - $("#dag_dl_logs").hide(); - $("#dag_redir_logs").hide(); - if (tryNumber > 0 && !isMapped) { - $("#dag_dl_logs").show(); - if (showExternalLogRedirect) { - $("#dag_redir_logs").show(); - } - } - - updateModalUrls({ - executionDate, - subDagId, - taskId, - mapIndex, - dagRunId, - }); - - $("#try_index > li").remove(); - $("#redir_log_try_index > li").remove(); - const startIndex = tryNumber > 2 ? 0 : 1; - - const query = new URLSearchParams({ - dag_id: dagId, - task_id: taskId, - execution_date: executionDate, - metadata: "null", - }); - if (mapIndex !== undefined) { - query.set("map_index", mapIndex); - } - for (let index = startIndex; index < tryNumber; index += 1) { - let showLabel = index; - if (index !== 0) { - query.set("try_number", index); - } else { - showLabel = "All"; - } - - $("#try_index").append(`<li role="presentation" style="display:inline"> - <a href="${logsWithMetadataUrl}?${query}&format=file"> ${showLabel} </a> - </li>`); - - if (index !== 0 || showExternalLogRedirect) { - $("#redir_log_try_index") - .append(`<li role="presentation" style="display:inline"> - <a href="${externalLogUrl}?${query}"> ${showLabel} </a> - </li>`); - } - } - query.delete("try_number"); - - if (!isMapped && extraLinks && extraLinks.length > 0) { - const markupArr = []; - extraLinks.sort(); - $.each(extraLinks, (i, link) => { - query.set("link_name", link); - const externalLink = $( - '<a href="#" class="btn btn-primary disabled"></a>' - ); - const linkTooltip = $( - '<span class="tool-tip" data-toggle="tooltip" style="padding-right: 2px; padding-left: 3px" data-placement="top" ' + - 'title="link not yet available"></span>' - ); - linkTooltip.append(externalLink); - externalLink.text(link); - - $.ajax({ - url: `${extraLinksUrl}?${query}`, - cache: false, - success(data) { - externalLink.attr("href", data.url); - // open absolute (external) links in a new tab/window and relative (local) links - // directly - if (/^(?:[a-z]+:)?\/\//.test(data.url)) { - externalLink.attr("target", "_blank"); - } - externalLink.removeClass("disabled"); - linkTooltip.tooltip("disable"); - }, - error(data) { - linkTooltip - .tooltip("hide") - .attr("title", data.responseJSON.error) - .tooltip("fixTitle"); - }, - }); - - markupArr.push(linkTooltip); - }); - - const extraLinksSpan = $("#extra_links"); - extraLinksSpan.prev("hr").show(); - extraLinksSpan.append(markupArr).show(); - extraLinksSpan.find('[data-toggle="tooltip"]').tooltip(); - } - - // Switch the modal from a mapped task summary to a specific mapped task instance - function switchMapItem() { - const mi = $(this).attr("data-mapIndex"); - if (mi === "all") { - callModal({ - taskId, - executionDate, - dagRunId, - extraLinks, - mapIndex: -1, - isMapped: true, - mappedStates, - }); - } else { - callModal({ - taskId, - executionDate, - dagRunId, - extraLinks, - mapIndex: mi, - }); - } - } - - // Task Instance Modal actions - function submit(e) { - e.preventDefault(); - const form = $(this).get(0); - if (dagRunId || executionDate) { - if (form.dag_run_id) { - form.dag_run_id.value = dagRunId; - } - if (form.execution_date) { - form.execution_date.value = executionDate; - } - form.origin.value = window.location; - if (form.task_id) { - form.task_id.value = taskId; - } - if (form.map_index && mapIndex >= 0) { - form.map_index.value = mapIndex; - } else if (form.map_index) { - form.map_index.remove(); - } - form.action = $(this).data("action"); - form.submit(); - } - } - - $("form[data-action]").on("submit", submit); - $(".map_index_item").on("click", switchMapItem); -} - -export default callModal; diff --git a/airflow/www/static/js/cluster-activity/nav/FilterBar.tsx b/airflow/www/static/js/cluster-activity/nav/FilterBar.tsx index 10cc6813f1..6cae7854f1 100644 --- a/airflow/www/static/js/cluster-activity/nav/FilterBar.tsx +++ b/airflow/www/static/js/cluster-activity/nav/FilterBar.tsx @@ -39,7 +39,9 @@ const FilterBar = () => { const startDate = moment(filters.startDate); // @ts-ignore const endDate = moment(filters.endDate); + // @ts-ignore const formattedStartDate = startDate.tz(timezone).format(isoFormatWithoutTZ); + // @ts-ignore const formattedEndDate = endDate.tz(timezone).format(isoFormatWithoutTZ); const inputStyles = { backgroundColor: "white", size: "lg" }; diff --git a/airflow/www/static/js/components/Time.test.tsx b/airflow/www/static/js/components/Time.test.tsx index 92ab8b39af..faef2a0eb3 100644 --- a/airflow/www/static/js/components/Time.test.tsx +++ b/airflow/www/static/js/components/Time.test.tsx @@ -17,12 +17,10 @@ * under the License. */ -/* global describe, test, expect, document, CustomEvent */ +/* global moment, describe, test, expect, document, CustomEvent */ import React from "react"; import { render, fireEvent, act } from "@testing-library/react"; -import moment from "moment-timezone"; - import { defaultFormatWithTZ, TimezoneEvent } from "src/datetime_utils"; import { Wrapper } from "src/utils/testUtils"; @@ -35,6 +33,7 @@ describe("Test Time and TimezoneProvider", () => { wrapper: Wrapper, }); + // @ts-ignore const utcTime = getByText(moment.utc(now).format(defaultFormatWithTZ)); expect(utcTime).toBeDefined(); expect(utcTime.title).toBeFalsy(); @@ -43,15 +42,18 @@ describe("Test Time and TimezoneProvider", () => { test("Displays moment default tz, includes UTC date in title", () => { const now = new Date(); const tz = "US/Samoa"; + // @ts-ignore moment.tz.setDefault(tz); const { getByText } = render(<Time dateTime={now.toISOString()} />, { wrapper: Wrapper, }); + // @ts-ignore const samoaTime = getByText(moment(now).tz(tz).format(defaultFormatWithTZ)); expect(samoaTime).toBeDefined(); expect(samoaTime.title).toEqual( + // @ts-ignore moment.utc(now).format(defaultFormatWithTZ) ); }); @@ -63,6 +65,7 @@ describe("Test Time and TimezoneProvider", () => { { wrapper: Wrapper } ); + // @ts-ignore const utcTime = queryByText(moment.utc(now).format(defaultFormatWithTZ)); expect(utcTime).toBeDefined(); @@ -76,9 +79,11 @@ describe("Test Time and TimezoneProvider", () => { expect(utcTime).toBeNull(); const estTime = getByText( + // @ts-ignore moment(now).tz("EST").format(defaultFormatWithTZ) ); expect(estTime).toBeDefined(); + // @ts-ignore expect(estTime.title).toEqual(moment.utc(now).format(defaultFormatWithTZ)); }); }); diff --git a/airflow/www/static/js/components/Time.tsx b/airflow/www/static/js/components/Time.tsx index c9765d7f05..8c0e2143a8 100644 --- a/airflow/www/static/js/components/Time.tsx +++ b/airflow/www/static/js/components/Time.tsx @@ -17,8 +17,9 @@ * under the License. */ +/* global moment */ + import React from "react"; -import moment from "moment-timezone"; import { useTimezone } from "src/context/timezone"; import { defaultFormatWithTZ } from "src/datetime_utils"; @@ -30,11 +31,14 @@ interface Props { const Time = ({ dateTime, format = defaultFormatWithTZ }: Props) => { const { timezone } = useTimezone(); + // @ts-ignore const time = moment(dateTime); if (!dateTime || !time.isValid()) return null; + // @ts-ignore const formattedTime = time.tz(timezone).format(format); + // @ts-ignore const utcTime = time.tz("UTC").format(defaultFormatWithTZ); return ( diff --git a/airflow/www/static/js/dag.js b/airflow/www/static/js/dag.js index 787c483719..c3d0cd7475 100644 --- a/airflow/www/static/js/dag.js +++ b/airflow/www/static/js/dag.js @@ -41,6 +41,31 @@ const setNextDatasets = (datasets, error) => { nextDatasetsError = error; }; +// Check if there is a highlighted tab and change the active nav button +const onTabChange = () => { + const urlParams = new URLSearchParams(window.location.search); + const isGrid = window.location.href.includes(`${dagId}/grid`); + const tab = urlParams.get("tab"); + const gridNav = document.getElementById("grid-nav"); + const graphNav = document.getElementById("graph-nav"); + const ganttNav = document.getElementById("gantt-nav"); + if (isGrid) { + if (tab === "graph") { + gridNav.classList.remove("active"); + ganttNav.classList.remove("active"); + graphNav.classList.add("active"); + } else if (tab === "gantt") { + gridNav.classList.remove("active"); + graphNav.classList.remove("active"); + ganttNav.classList.add("active"); + } else { + graphNav.classList.remove("active"); + ganttNav.classList.remove("active"); + gridNav.classList.add("active"); + } + } +}; + // Pills highlighting $(window).on("load", function onLoad() { $(`a[href*="${this.location.pathname}"]`).parent().addClass("active"); @@ -50,6 +75,32 @@ $(window).on("load", function onLoad() { if (!singleDatasetUri) { getDatasetTooltipInfo(dagId, run, setNextDatasets); } + + onTabChange(); +}); + +// Dispatch an event whenever history changes that we can then listen to +const LOCATION_CHANGE = "locationchange"; +(function dispatchLocationEvent() { + const { pushState, replaceState } = window.history; + + window.history.pushState = (...args) => { + pushState.apply(window.history, args); + window.dispatchEvent(new Event(LOCATION_CHANGE)); + }; + + window.history.replaceState = (...args) => { + replaceState.apply(window.history, args); + window.dispatchEvent(new Event(LOCATION_CHANGE)); + }; + + window.addEventListener("popstate", () => { + window.dispatchEvent(new Event(LOCATION_CHANGE)); + }); +})(); + +window.addEventListener(LOCATION_CHANGE, () => { + onTabChange(); }); $("#pause_resume").on("change", function onChange() { diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts index 21e215fe1e..80b534ccd5 100644 --- a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts +++ b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts @@ -82,6 +82,7 @@ export const parseLogs = ( // @ts-ignore const localDateTime = moment .utc(dateTime) + // @ts-ignore .tz(timezone) .format(defaultFormatWithTZ); parsedLine = line.replace(dateTime, localDateTime); diff --git a/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx b/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx index 2ac20ea3cc..a02df41975 100644 --- a/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx +++ b/airflow/www/static/js/dag/grid/dagRuns/index.test.tsx @@ -17,11 +17,10 @@ * under the License. */ -/* global describe, test, expect, jest */ +/* global describe, test, expect, jest, moment */ import React from "react"; import { render } from "@testing-library/react"; -import moment from "moment-timezone"; import { TableWrapper } from "src/utils/testUtils"; import * as useGridDataModule from "src/api/useGridData"; @@ -105,6 +104,7 @@ describe("Test DagRuns", () => { expect(getByText("00:02:53")).toBeInTheDocument(); expect(getByText("00:01:26")).toBeInTheDocument(); expect( + // @ts-ignore queryByText(moment.utc(dagRuns[0].executionDate).format("MMM DD, HH:mm")) ).toBeNull(); @@ -124,6 +124,7 @@ describe("Test DagRuns", () => { ); const { getByText } = render(<DagRuns />, { wrapper: TableWrapper }); expect( + // @ts-ignore getByText(moment.utc(datestring).format("MMM DD, HH:mm")) ).toBeInTheDocument(); spy.mockRestore(); @@ -142,6 +143,7 @@ describe("Test DagRuns", () => { ); const { queryAllByText } = render(<DagRuns />, { wrapper: TableWrapper }); expect( + // @ts-ignore queryAllByText(moment.utc(datestring).format("MMM DD, HH:mm")) ).toHaveLength(1); spy.mockRestore(); @@ -160,6 +162,7 @@ describe("Test DagRuns", () => { ); const { queryAllByText } = render(<DagRuns />, { wrapper: TableWrapper }); expect( + // @ts-ignore queryAllByText(moment.utc(datestring).format("MMM DD, HH:mm")) ).toHaveLength(2); spy.mockRestore(); diff --git a/airflow/www/static/js/dag/nav/FilterBar.tsx b/airflow/www/static/js/dag/nav/FilterBar.tsx index 9078fe705a..fc3a56f1d5 100644 --- a/airflow/www/static/js/dag/nav/FilterBar.tsx +++ b/airflow/www/static/js/dag/nav/FilterBar.tsx @@ -48,6 +48,7 @@ const FilterBar = () => { const { timezone } = useTimezone(); // @ts-ignore const time = moment(filters.baseDate); + // @ts-ignore const formattedTime = time.tz(timezone).format(isoFormatWithoutTZ); const inputStyles = { backgroundColor: "white", size: "lg" }; diff --git a/airflow/www/static/js/graph.js b/airflow/www/static/js/graph.js deleted file mode 100644 index b61639d3a8..0000000000 --- a/airflow/www/static/js/graph.js +++ /dev/null @@ -1,828 +0,0 @@ -/* eslint-disable no-underscore-dangle */ -/* eslint-disable no-use-before-define */ -/*! - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - global d3, document, nodes, taskInstances, tasks, edges, dagreD3, localStorage, $, - autoRefreshInterval, moment, convertSecsToHumanReadable, priority -*/ - -import { getMetaValue, finalStatesMap } from "./utils"; -import { escapeHtml } from "./main"; -import tiTooltip, { taskNoInstanceTooltip } from "./task_instances"; -import callModal from "./callModal"; - -// dagId comes from dag.html -const dagId = getMetaValue("dag_id"); -const executionDate = getMetaValue("execution_date"); -const dagRunId = getMetaValue("dag_run_id"); -const arrange = getMetaValue("arrange"); -const taskInstancesUrl = getMetaValue("task_instances_url"); -const isSchedulerRunning = getMetaValue("is_scheduler_running"); - -// This maps the actual taskId to the current graph node id that contains the task -// (because tasks may be grouped into a group node) -const mapTaskToNode = new Map(); - -// Below variables are being used in dag.js - -const getTaskInstanceURL = `${taskInstancesUrl}?dag_id=${encodeURIComponent( - dagId -)}&execution_date=${encodeURIComponent(executionDate)}`; - -const stateFocusMap = { - success: false, - running: false, - failed: false, - skipped: false, - upstream_failed: false, - up_for_reschedule: false, - up_for_retry: false, - queued: false, - deferred: false, - no_status: false, -}; - -const checkRunState = () => { - const states = Object.values(taskInstances).map((ti) => ti.state); - return !states.some( - (state) => - ["success", "failed", "upstream_failed", "skipped", "removed"].indexOf( - state - ) === -1 - ); -}; - -const taskTip = d3 - .tip() - .attr("class", "tooltip d3-tip") - .html((toolTipHtml) => toolTipHtml); - -// Preparation of DagreD3 data structures -// "compound" is set to true to make use of clusters to display TaskGroup. -const g = new dagreD3.graphlib.Graph({ compound: true }) - .setGraph({ - nodesep: 30, - ranksep: 15, - rankdir: arrange, - }) - .setDefaultEdgeLabel(() => ({ lineInterpolate: "basis" })); - -const render = dagreD3.render(); -const svg = d3.select("#graph-svg"); -let innerSvg = d3.select("#graph-svg g"); - -// We modify the label of task map nodes to include the brackets and a count of mapped tasks -// returns true if at least one node is changed -const updateNodeLabels = (node, instances) => { - let haveLabelsChanged = false; - let { label } = node.value; - - const isGroupMapped = node.value.isMapped; - const isTaskMapped = tasks[node.id] && tasks[node.id].is_mapped; - - if (isTaskMapped || isGroupMapped) { - // get count from mapped_states or the first child's mapped_states - const id = isGroupMapped ? node.children[0].id : node.id; - const instance = instances[id]; - let count = " "; - - // TODO: update this count for when we can nest mapped tasks inside of mapped task groups - if (instance && instance.mapped_states) { - count = instance.mapped_states.length; - } - - if (!label.includes(`[${count}]`)) { - label = `${label} [${count}]`; - } - } - if (g.node(node.id) && g.node(node.id).label !== label) { - g.node(node.id).label = label; - haveLabelsChanged = true; - } - - if (node.children) { - // Iterate through children and return true if at least one has been changed - const updatedNodes = node.children.map((n) => - updateNodeLabels(n, instances) - ); - return updatedNodes.some((changed) => changed); - } - - return haveLabelsChanged; -}; - -// Remove the node with this nodeId from g. -function removeNode(nodeId) { - if (g.hasNode(nodeId)) { - const node = g.node(nodeId); - if (node.children !== undefined) { - // If the child is an expanded group node, remove children too. - node.children.forEach((child) => { - removeNode(child.id); - }); - } - } - g.removeNode(nodeId); -} - -// Collapse the children of the given group node. -function collapseGroup(nodeId, node) { - // Remove children nodes - node.children.forEach((child) => { - removeNode(child.id); - }); - // Map task that are under this node to this node's id - - getChildrenIds(node).forEach((childId) => mapTaskToNode.set(childId, nodeId)); - - // eslint-disable-next-line no-param-reassign - node = g.node(nodeId); - - // Set children edges onto the group edge - edges.forEach((edge) => { - const sourceId = mapTaskToNode.get(edge.source_id); - const targetId = mapTaskToNode.get(edge.target_id); - if (sourceId !== targetId && !g.hasEdge(sourceId, targetId)) { - g.setEdge(sourceId, targetId, { - curve: d3.curveBasis, - arrowheadClass: "arrowhead", - }); - } - }); - - draw(); - focusGroup(nodeId); - - removeExpandedGroup(nodeId, node); -} - -// Update the page to show the latest DAG. -function draw() { - innerSvg.remove(); - innerSvg = svg.append("g"); - // Run the renderer. This is what draws the final graph. - innerSvg.call(render, g); - innerSvg.call(taskTip); - - // When an expanded group is clicked, collapse it. - d3.selectAll("g.cluster").on("click", (nodeId) => { - if (d3.event.defaultPrevented) return; - const node = g.node(nodeId); - collapseGroup(nodeId, node); - }); - // When a node is clicked, action depends on the node type. - d3.selectAll("g.node").on("click", (nodeId) => { - const node = g.node(nodeId); - if (node.children !== undefined && Object.keys(node.children).length > 0) { - // A group node - if (d3.event.defaultPrevented) return; - expandGroup(nodeId, node); - updateNodeLabels(nodes, taskInstances); - draw(); - focusGroup(nodeId); - } else if (nodeId in taskInstances) { - // A task node - const task = tasks[nodeId]; - const tryNumber = taskInstances[nodeId].try_number || 0; - const mappedStates = taskInstances[nodeId].mapped_states || []; - - callModal({ - taskId: nodeId, - executionDate, - extraLinks: task.extra_links, - tryNumber, - isSubDag: task.task_type === "SubDagOperator", - dagRunId, - mapIndex: task.map_index, - isMapped: task.is_mapped || !!taskInstances[nodeId].mapped_states, - mappedStates, - }); - } - }); - - d3.selectAll("g.node").on("mouseover", function mousover(d) { - d3.select(this).selectAll("rect").attr("data-highlight", "highlight"); - highlightNodes(g.predecessors(d)); - highlightNodes(g.successors(d)); - const adjacentNodeNames = [d, ...g.predecessors(d), ...g.successors(d)]; - - d3.selectAll("g.nodes g.node") - .filter((x) => !adjacentNodeNames.includes(x)) - .attr("data-highlight", "fade"); - - d3.selectAll("g.edgePath")[0].forEach((x) => { - const val = g.nodeEdges(d).includes(x.__data__) ? "highlight" : "fade"; - d3.select(x).attr("data-highlight", val); - }); - d3.selectAll("g.edgeLabel")[0].forEach((x) => { - if (!g.nodeEdges(d).includes(x.__data__)) { - d3.select(x).attr("data-highlight", "fade"); - } - }); - }); - - d3.selectAll("g.node").on("mouseout", function mouseout(d) { - d3.select(this).selectAll("rect, circle").attr("data-highlight", null); - unHighlightNodes(g.predecessors(d)); - unHighlightNodes(g.successors(d)); - d3.selectAll("g.node, g.edgePath, g.edgeLabel").attr( - "data-highlight", - null - ); - localStorage.removeItem(focusedGroupKey(dagId)); - }); - updateNodesStates(taskInstances); - setUpZoomSupport(); -} - -let zoom = null; -const maxZoom = 0.3; - -function setUpZoomSupport() { - // Set up zoom support for Graph - zoom = d3.behavior.zoom().on("zoom", () => { - innerSvg.attr( - "transform", - `translate(${d3.event.translate})scale(${d3.event.scale})` - ); - }); - svg.call(zoom); - - // Centering the DAG on load - // Get Dagre Graph dimensions - const graphWidth = g.graph().width; - const graphHeight = g.graph().height; - const { width, height } = svg.node().viewBox.animVal; - const padding = width * 0.05; - - // Calculate applicable scale for zoom - - const zoomScale = - Math.min(Math.min(width / graphWidth, height / graphHeight), maxZoom) * 0.8; - - zoom.translate([width / 2 - (graphWidth * zoomScale) / 2 + padding, padding]); - zoom.scale(zoomScale); - zoom.event(innerSvg); - zoom.scaleExtent([0, maxZoom]); -} - -function highlightNodes(nodes) { - nodes.forEach((nodeid) => { - const myNode = g.node(nodeid).elem; - d3.select(myNode) - .selectAll("rect, circle") - .attr("data-highlight", "highlight"); - }); -} - -function unHighlightNodes(nodes) { - nodes.forEach((nodeid) => { - const myNode = g.node(nodeid).elem; - d3.select(myNode).selectAll("rect, circle").attr("data-highlight", null); - }); -} - -d3.selectAll(".js-state-legend-item") - .on("mouseover", function mouseover() { - if (!stateIsSet()) { - const state = $(this).data("state"); - focusState(state); - } - }) - .on("mouseout", () => { - if (!stateIsSet()) { - clearFocus(); - } - }); - -d3.selectAll(".js-state-legend-item").on("click", function click() { - const state = $(this).data("state"); - - clearFocus(); - if (!stateFocusMap[state]) { - const color = d3.select(this).style("border-color"); - focusState(state, this, color); - setFocusMap(state); - } else { - setFocusMap(); - d3.selectAll(".js-state-legend-item").style("background-color", null); - } -}); - -// Returns true if a node's id or its children's id matches searchText -function nodeMatches(nodeId, searchText) { - if (nodeId.indexOf(searchText) > -1) return true; - - // The node's own id does not match, it may have children that match - const node = g.node(nodeId); - if (node.children) { - const children = getChildrenIds(node); - return !!children.find((child) => child.indexOf(searchText) > -1); - } - return false; -} - -d3.select("#searchbox").on("keyup", () => { - const s = document.getElementById("searchbox").value; - - if (s === "") return; - - let match = null; - - if (stateIsSet()) { - clearFocus(); - setFocusMap(); - } - - d3.selectAll("g.nodes g.node").filter(function highlight(d) { - if (s === "") { - d3.selectAll("g.edgePaths, g.edgeLabel").attr("data-highlight", null); - d3.select(this).attr("data-highlight", null); - } else { - d3.selectAll("g.edgePaths, g.edgeLabel").attr("data-highlight", "fade"); - if (nodeMatches(d, s)) { - if (!match) match = this; - d3.select(this).attr("data-highlight", null); - } else { - d3.select(this).attr("data-highlight", "fade"); - } - } - // We don't actually use the returned results from filter - return null; - }); - - // This moves the matched node to the center of the graph area - if (match) { - focusGroup(match.id, false); - } -}); - -function clearFocus() { - d3.selectAll("g.node, g.edgePaths, g.edgeLabel").attr("data-highlight", null); - localStorage.removeItem(focusedGroupKey(dagId)); -} - -function focusState(state, node, color) { - d3.selectAll("g.node, g.edgePaths, g.edgeLabel").attr( - "data-highlight", - "fade" - ); - d3.selectAll(`g.node.${state}`).attr("data-highlight", null); - d3.selectAll(`g.node.${state} rect`).attr("data-highlight", null); - d3.select(node).style("background-color", color); -} - -function setFocusMap(state) { - Object.keys(stateFocusMap).forEach((key) => { - if ({}.hasOwnProperty.call(stateFocusMap, key)) { - stateFocusMap[key] = false; - } - }); - if (state != null) { - stateFocusMap[state] = true; - } -} - -const stateIsSet = () => - !!Object.keys(stateFocusMap).find((key) => stateFocusMap[key]); - -let refreshInterval; - -function startOrStopRefresh() { - if ($("#auto_refresh").is(":checked")) { - refreshInterval = setInterval(() => { - handleRefresh(); - }, autoRefreshInterval * 1000); - } else { - clearInterval(refreshInterval); - } -} - -// pause autorefresh when the page is not active -const handleVisibilityChange = () => { - if (document.hidden) { - clearInterval(refreshInterval); - } else { - initRefresh(); - } -}; - -document.addEventListener("visibilitychange", handleVisibilityChange); - -let prevTis; - -function handleRefresh() { - $("#loading-dots").css("display", "inline-block"); - $.get(getTaskInstanceURL) - .done((tis) => { - // only refresh if the data has changed - if (prevTis !== tis) { - // eslint-disable-next-line no-global-assign - updateNodesStates(tis); - - // Only redraw the graph if labels have changed - const haveLabelsChanged = updateNodeLabels(nodes, tis); - if (haveLabelsChanged) draw(); - - // end refresh if all states are final - const isFinal = checkRunState(); - if (isFinal) { - $("#auto_refresh").prop("checked", false); - clearInterval(refreshInterval); - } - } - prevTis = tis; - setTimeout(() => { - $("#loading-dots").hide(); - }, 500); - $("#error").hide(); - }) - .fail((response, textStatus, err) => { - const description = - (response.responseJSON && response.responseJSON.error) || - "Something went wrong."; - $("#error_msg").text(`${textStatus}: ${err} ${description}`); - $("#error").show(); - setTimeout(() => { - $("#loading-dots").hide(); - }, 500); - $("#chart_section").hide(1000); - $("#datatable_section").hide(1000); - }); -} - -$("#auto_refresh").change(() => { - if ($("#auto_refresh").is(":checked")) { - // Run an initial refresh before starting interval if manually turned on - handleRefresh(); - localStorage.removeItem("disableAutoRefresh"); - } else { - localStorage.setItem("disableAutoRefresh", "true"); - } - startOrStopRefresh(); -}); - -function initRefresh() { - const isDisabled = localStorage.getItem("disableAutoRefresh"); - const isFinal = checkRunState(); - $("#auto_refresh").prop( - "checked", - !(isDisabled || isFinal) && isSchedulerRunning === "True" - ); - startOrStopRefresh(); - d3.select("#refresh_button").on("click", () => handleRefresh()); -} - -// Generate tooltip for a group node -function groupTooltip(node, tis) { - const numMap = finalStatesMap(); - - let minStart; - let maxEnd; - - if (node.isMapped) { - const firstChildId = node.children[0].id; - const mappedLength = tis[firstChildId].mapped_states.length; - [...Array(mappedLength).keys()].forEach((mapIndex) => { - const groupStates = getChildrenIds(node).map( - (child) => tis[child].mapped_states[mapIndex] - ); - const overallState = - priority.find((state) => groupStates.includes(state)) || "no_status"; - if (numMap.has(overallState)) - numMap.set(overallState, numMap.get(overallState) + 1); - }); - } else { - getChildrenIds(node).forEach((child) => { - if (child in tis) { - const ti = tis[child]; - if (!minStart || moment(ti.start_date).isBefore(minStart)) { - minStart = moment(ti.start_date); - } - if (!maxEnd || moment(ti.end_date).isAfter(maxEnd)) { - maxEnd = moment(ti.end_date); - } - const stateKey = ti.state == null ? "no_status" : ti.state; - if (numMap.has(stateKey)) - numMap.set(stateKey, numMap.get(stateKey) + 1); - } - }); - } - - const groupDuration = convertSecsToHumanReadable( - moment(maxEnd).diff(minStart, "second") - ); - const tooltipText = node.tooltip ? `<p>${node.tooltip}</p>` : ""; - - let tt = ` - ${tooltipText} - <strong>Duration:</strong> ${groupDuration} <br><br> - `; - numMap.forEach((key, val) => { - if (key > 0) { - tt += `<strong>${escapeHtml(val)}:</strong> ${key} <br>`; - } - }); - - return tt; -} - -// Assigning css classes based on state to nodes -// Initiating the tooltips -function updateNodesStates(tis) { - g.nodes().forEach((nodeId) => { - const node = g.node(nodeId); - const { elem } = node; - const taskId = nodeId; - - if (elem) { - const classes = `node enter ${getNodeState(nodeId, tis)}`; - elem.setAttribute("class", classes); - elem.setAttribute("data-toggle", "tooltip"); - - elem.onmouseover = (evt) => { - let tt; - if (taskId in tis) { - tt = tiTooltip(tis[taskId], tasks[taskId]); - } else if (node.children) { - tt = groupTooltip(node, tis); - } else if (taskId in tasks) { - tt = taskNoInstanceTooltip(taskId, tasks[taskId]); - elem.setAttribute("class", `${classes} not-allowed`); - } - if (tt) taskTip.show(tt, evt.target); // taskTip is defined in graph.html - }; - elem.onmouseout = taskTip.hide; - elem.onclick = taskTip.hide; - } - }); -} - -// Returns list of children id of the given task group -function getChildrenIds(group) { - const children = []; - Object.values(group.children).forEach((value) => { - if (value.children === undefined) { - // node - children.push(value.id); - } else { - // group - const subGroupChildren = getChildrenIds(value); - subGroupChildren.forEach((id) => children.push(id)); - } - }); - return children; -} - -// Return list of all task group ids in the given task group including the given group itself. -function getAllGroupIds(group) { - const children = [group.id]; - - Object.entries(group.children).forEach(([, val]) => { - if (val.children !== undefined) { - // group - const subGroupChildren = getAllGroupIds(val); - subGroupChildren.forEach((id) => children.push(id)); - } - }); - return children; -} - -// Return the state for the node based on the state of its taskinstance or that of its -// children if it's a group node -function getNodeState(nodeId, tis) { - const node = g.node(nodeId); - - if (node.children === undefined) { - if (nodeId in tis) { - return tis[nodeId].state || "no_status"; - } - return "no_status"; - } - const children = getChildrenIds(node); - - const childrenStates = new Set(); - children.forEach((taskId) => { - if (taskId in tis) { - const { state } = tis[taskId]; - childrenStates.add(state == null ? "no_status" : state); - } - }); - - return priority.find((state) => childrenStates.has(state)) || "no_status"; -} - -// Returns the key used to store expanded task group ids in localStorage -function expandedGroupsKey() { - return `expandedGroups_${dagId}`; -} - -// Returns the key used to store the focused task group id in localStorage -function focusedGroupKey() { - return `focused_group_${dagId}`; -} - -// Focus the graph on the expanded/collapsed node -function focusGroup(nodeId, followMouse = true) { - if (nodeId != null && zoom != null) { - const { x, y } = g.node(nodeId); - // This is the total canvas size. - const { width, height } = svg.node().viewBox.animVal; - - // This is the size of the node or the cluster (i.e. group) - let rect = d3 - .selectAll("g.node") - .filter((n) => n === nodeId) - .select("rect"); - if (rect.empty()) - rect = d3 - .selectAll("g.cluster") - .filter((n) => n === nodeId) - .select("rect"); - - const [mouseX, mouseY] = d3.mouse(svg.node()); - - // Is there a better way to get nodeWidth and nodeHeight ? - const [nodeWidth, nodeHeight] = [ - rect[0][0].attributes.width.value, - rect[0][0].attributes.height.value, - ]; - - // Calculate zoom scale to fill most of the canvas with the node/cluster in focus. - const scale = - Math.min(Math.min(width / nodeWidth, height / nodeHeight), maxZoom) * 0.4; - - // Move the graph so that the node that was expanded/collapsed is centered around - // the mouse click. - const [toX, toY] = followMouse ? [mouseX, mouseY] : [width / 2, height / 5]; - const [deltaX, deltaY] = [ - toX - x * scale, - toY + (nodeHeight / 2 - y) * scale, - ]; - zoom.translate([deltaX, deltaY]); - zoom.scale(scale); - zoom.event(innerSvg); - - const children = new Set(g.children(nodeId)); - // Set data attr to highlight the focused group (via CSS). - d3.selectAll("g.nodes g.node").forEach(function cssHighlight(d) { - if (d === nodeId || children.has(d)) { - d3.select(this).attr("data-highlight", null); - } else { - d3.select(this).attr("data-highlight", "fade"); - } - }); - - localStorage.setItem(focusedGroupKey(dagId), nodeId); - } -} - -// Expands a group node -function expandGroup(nodeId, node) { - node.children.forEach((val) => { - // Set children nodes - g.setNode(val.id, val.value); - mapTaskToNode.set(val.id, val.id); - g.node(val.id).id = val.id; - if (val.children !== undefined) { - // Set children attribute so that the group can be expanded later when needed. - const groupNode = g.node(val.id); - groupNode.children = val.children; - // Map task that are under this node to this node's id - getChildrenIds(val).forEach((childId) => - mapTaskToNode.set(childId, val.id) - ); - } - // Only call setParent if node is not the root node. - if (nodeId != null) g.setParent(val.id, nodeId); - }); - - // Add edges - edges.forEach((edge) => { - const sourceId = mapTaskToNode.get(edge.source_id); - const targetId = mapTaskToNode.get(edge.target_id); - if ( - sourceId !== targetId && - !g.hasEdge(sourceId, targetId) && - sourceId && - targetId - ) { - g.setEdge(sourceId, targetId, { - curve: d3.curveBasis, - arrowheadClass: "arrowhead", - label: edge.label, - }); - } - }); - - g.edges().forEach((edge) => { - // Remove edges that were associated with the expanded group node.. - if (nodeId === edge.v || nodeId === edge.w) { - g.removeEdge(edge.v, edge.w); - } - }); - - saveExpandedGroup(nodeId); -} - -function getSavedGroups() { - let expandedGroups; - try { - expandedGroups = new Set( - JSON.parse(localStorage.getItem(expandedGroupsKey(dagId))) - ); - } catch { - expandedGroups = new Set(); - } - - return expandedGroups; -} - -// Clean up invalid group_ids from saved_group_ids (e.g. due to DAG changes) -function pruneInvalidSavedGroupIds() { - // All the groupIds in the whole DAG - const allGroupIds = new Set(getAllGroupIds(nodes)); - let expandedGroups = getSavedGroups(dagId); - expandedGroups = Array.from(expandedGroups).filter((groupId) => - allGroupIds.has(groupId) - ); - localStorage.setItem( - expandedGroupsKey(dagId), - JSON.stringify(expandedGroups) - ); -} - -// Remember the expanded groups in local storage so that it can be used -// to restore the expanded state of task groups. -function saveExpandedGroup(nodeId) { - // expandedGroups is a Set - const expandedGroups = getSavedGroups(dagId); - expandedGroups.add(nodeId); - localStorage.setItem( - expandedGroupsKey(dagId), - JSON.stringify(Array.from(expandedGroups)) - ); -} - -// Remove the nodeId from the expanded state -function removeExpandedGroup(nodeId, node) { - const expandedGroups = getSavedGroups(dagId); - const childGroupIds = getAllGroupIds(node); - childGroupIds.forEach((childId) => expandedGroups.delete(childId)); - localStorage.setItem( - expandedGroupsKey(dagId), - JSON.stringify(Array.from(expandedGroups)) - ); -} - -// Restore previously expanded task groups -function expandSavedGroups(expandedGroups, node) { - if (node.children === undefined) return; - - node.children.forEach((childNode) => { - if (expandedGroups.has(childNode.id)) { - expandGroup(childNode.id, g.node(childNode.id)); - - expandSavedGroups(expandedGroups, childNode); - } - }); -} - -pruneInvalidSavedGroupIds(); -const focusNodeId = localStorage.getItem(focusedGroupKey(dagId)); -const expandedGroups = getSavedGroups(dagId); - -// Always expand the root node -expandGroup(null, nodes); - -// Expand the node that were previously expanded -expandSavedGroups(expandedGroups, nodes); - -// Draw once after all groups have been expanded -updateNodeLabels(nodes, taskInstances); -draw(); - -// Restore focus (if available) -if (g.hasNode(focusNodeId)) { - focusGroup(focusNodeId); -} - -initRefresh(); diff --git a/airflow/www/static/js/task_instances.js b/airflow/www/static/js/task_instances.js index abadfaab00..47a5dccb6f 100644 --- a/airflow/www/static/js/task_instances.js +++ b/airflow/www/static/js/task_instances.js @@ -17,7 +17,7 @@ * under the License. */ -/* global window, moment, convertSecsToHumanReadable */ +/* global moment, convertSecsToHumanReadable */ // We don't re-import moment again, otherwise webpack will include it twice in the bundle! import { escapeHtml } from "./main"; @@ -147,50 +147,3 @@ export default function tiTooltip(ti, task, { includeTryNumber = false } = {}) { tt += generateTooltipDateTimes(ti.start_date, ti.end_date, dagTZ || "UTC"); return tt; } - -export function taskNoInstanceTooltip(taskId, task) { - let tt = ""; - if (taskId) { - tt += `Task_id: ${escapeHtml(taskId)}<br>`; - } - if (task.operator_name !== undefined) { - tt += `Operator: ${escapeHtml(task.operator_name)}<br>`; - } - tt += "<br><em>DAG has yet to run.</em>"; - return tt; -} - -export function taskQueuedStateTooltip(ti) { - let tt = ""; - tt += "<strong>Status:</strong> Queued<br><br>"; - if (ti.task_id) { - tt += `Task_id: ${escapeHtml(ti.task_id)}<br>`; - } - tt += `Run: ${formatDateTime(ti.execution_date)}<br>`; - if (ti.run_id !== undefined) { - tt += `Run Id: <nobr>${escapeHtml(ti.run_id)}</nobr><br>`; - } - if (ti.operator_name !== undefined) { - tt += `Operator: ${escapeHtml(ti.operator_name)}<br>`; - } - if (ti.start_date && ti.queued_dttm) { - const startDate = - ti.start_date instanceof moment ? ti.start_date : moment(ti.start_date); - const queuedDate = - ti.queued_dttm instanceof moment - ? ti.queued_dttm - : moment(ti.queued_dttm); - const duration = startDate.diff(queuedDate, "second", true); // Set the floating point result flag to true. - tt += `Duration: ${escapeHtml(convertSecsToHumanReadable(duration))}<br>`; - // dagTZ has been defined in dag.html - tt += generateTooltipDateTimes( - ti.queued_dttm, - ti.start_date, - dagTZ || "UTC" - ); - } - return tt; -} - -window.tiTooltip = tiTooltip; -window.taskNoInstanceTooltip = taskNoInstanceTooltip; diff --git a/airflow/www/templates/airflow/dag.html b/airflow/www/templates/airflow/dag.html index 754e17262a..2054433a1b 100644 --- a/airflow/www/templates/airflow/dag.html +++ b/airflow/www/templates/airflow/dag.html @@ -184,11 +184,11 @@ <div class="row"> <div class="col-md-10"> <ul class="nav nav-pills"> - <li><a href="{{ url_for('Airflow.grid', dag_id=dag.dag_id, num_runs=num_runs_arg, root=root, base_date=base_date_arg) }}"> + <li id="grid-nav"><a href="{{ url_for('Airflow.grid', dag_id=dag.dag_id, num_runs=num_runs_arg, root=root, base_date=base_date_arg) }}"> <span class="material-icons" aria-hidden="true">grid_on</span> Grid </a></li> - <li><a href="{{ url_for('Airflow.graph', dag_id=dag.dag_id, root=root, num_runs=num_runs_arg, base_date=base_date_arg, execution_date=execution_date_arg) }}"> + <li id="graph-nav"><a href="{{ url_for('Airflow.graph', dag_id=dag.dag_id, root=root, num_runs=num_runs_arg, base_date=base_date_arg, execution_date=execution_date_arg) }}"> <span class="material-icons" aria-hidden="true">account_tree</span> Graph</a></li> <li><a href="{{ url_for('Airflow.calendar', dag_id=dag.dag_id) }}"> @@ -204,7 +204,7 @@ <li><a href="{{ url_for('Airflow.landing_times', dag_id=dag.dag_id, days=30, root=root, num_runs=num_runs_arg, base_date=base_date_arg) }}"> <span class="material-icons" aria-hidden="true">flight_land</span> Landing Times</a></li> - <li><a href="{{ url_for('Airflow.gantt', dag_id=dag.dag_id, root=root, num_runs=num_runs_arg, base_date=base_date_arg, execution_date=execution_date_arg) }}"> + <li id="gantt-nav"><a href="{{ url_for('Airflow.gantt', dag_id=dag.dag_id, root=root, num_runs=num_runs_arg, base_date=base_date_arg, execution_date=execution_date_arg) }}"> <span class="material-icons" aria-hidden="true">vertical_distribute</span> Gantt</a></li> <li><a href="{{ url_for('Airflow.dag_details', dag_id=dag.dag_id) }}"> diff --git a/airflow/www/templates/airflow/graph.html b/airflow/www/templates/airflow/graph.html deleted file mode 100644 index 64b9a03c73..0000000000 --- a/airflow/www/templates/airflow/graph.html +++ /dev/null @@ -1,146 +0,0 @@ -{# - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -#} - -{% extends "airflow/dag.html" %} -{% from 'appbuilder/loading_dots.html' import loading_dots %} - -{% block page_title %}{{ dag.dag_id }} - Graph - {{ appbuilder.app_name }}{% endblock %} - -{% block head_meta %} - {{ super() }} - <meta name="dag_run_id" content="{{ dag_run_id }}"> - <meta name="execution_date" content="{{ execution_date }}"> - <meta name="arrange" content="{{ arrange }}"> - <meta name="task_instances_url" content="{{ url_for('Airflow.task_instances') }}"> -{% endblock %} - -{% block head_css %} - {{ super() }} - <link rel="stylesheet" type="text/css" href="{{ url_for_asset('graph.css') }}"> - <style type="text/css"> - {% for state, state_color in state_color_mapping.items() %} - g.node.{{state}} rect { - stroke: {{state_color}}; - } - {% endfor %} - </style> -{% endblock %} - -{% block content %} - {{ super() }} - <div class="row dag-view-tools"> - <div class="col-md-10"> - <form method="get" class="form-inline"> - <input type="hidden" name="root" value="{{ root }}"> - <input type="hidden" value="{{ dag.dag_id }}" name="dag_id"> - <input type="hidden" name="_csrf_token" value="{{ csrf_token() }}"> - <div class="form-group"> - <label class="sr-only" for="base_date">Base date</label> - <div class="input-group"> - {{ form.base_date(class_="form-control", disabled=not(dag.has_dag_runs())) }} - </div> - </div> - <div class="form-group"> - <label class="sr-only" for="num_runs">Number of runs</label> - <div class="input-group"> - <div class="input-group-addon">Runs</div> - {{ form.num_runs(class_="form-control", disabled=not(dag.has_dag_runs())) }} - </div> - </div> - <div class="form-group"> - <label class="sr-only" for="execution_date">Run</label> - <div class="input-group"> - <div class="input-group-addon">Run</div> - {{ form.execution_date(class_="form-control", disabled=not(dag.has_dag_runs())) }} - </div> - </div> - <div class="form-group"> - <label class="sr-only" for="arrange">Layout</label> - <div class="input-group"> - <div class="input-group-addon">Layout</div> - {{ form.arrange(class_="form-control") }} - </div> - </div> - <button type="submit" class="btn">Update</button> - {% if not dag.has_dag_runs() %}<span class="text-warning" style="margin-left:16px;">No DAG runs yet.</span>{% endif %} - </form> - </div> - <div class="col-md-2 text-right"> - <label class="sr-only" for="searchbox">Search</label> - <input type="search" class="form-control" id="searchbox" placeholder="Find Task…"> - </div> - </div> - - <div class="legend-row"> - <div> - {% for op in operators %}<span class="legend-item" style="color: {{ op.ui_fgcolor }};background: {{ op.ui_color }};"> - {{ op.operator_name }}</span>{% endfor %} - </div> - <div> - {% for state, state_color in state_color_mapping.items() %} - <span class="legend-item legend-item--interactive js-state-legend-item" data-state="{{state}}" style="border-color: {{state_color}};"> - {{state}} - </span> - {% endfor %} - <span class="legend-item legend-item--interactive legend-item--no-border js-state-legend-item" data-state="no_status" style="border-color:white;"> - no_status - </span> - </div> - </div> - <div id="error" style="display: none; margin-top: 10px;" class="alert alert-danger" role="alert"> - <span class="material-icons" aria-hidden="true">error</span> - <span id="error_msg">Oops.</span> - </div> - <br> - <div class="refresh-actions"> - {{ loading_dots(id='loading-dots', classes='refresh-loading') }} - <label class="switch-label"> - <input class="switch-input" id="auto_refresh" type="checkbox" {% if dag_run_state == 'running' %}checked{% endif %}> - <span class="switch" aria-hidden="true"></span> - Auto-refresh - </label> - <button class="btn btn-default btn-sm" id="refresh_button"> - <span class="material-icons" aria-hidden="true">refresh</span> - </button> - </div> - <div class="svg-wrapper"> - <div class="graph-svg-wrap"> - <svg id="graph-svg" viewBox="0 0 100 100"> - <g id="dig" transform="translate(20,20)"></g> - </svg> - </div> - </div> -{% endblock %} - -{% block tail %} - {{ super() }} - <script> - const nodes = {{ nodes|tojson }}; - const edges = {{ edges|tojson }}; - const tasks = {{ tasks|tojson }}; - let taskInstances = {{ task_instances|tojson }}; - const autoRefreshInterval = {{ auto_refresh_interval }}; - const priority = {{ state_priority|tojson }}; - </script> - <script src="{{ url_for_asset('d3.min.js') }}"></script> - <script src="{{ url_for_asset('dagre-d3.min.js') }}"></script> - <script src="{{ url_for_asset('d3-shape.min.js') }}"></script> - <script src="{{ url_for_asset('d3-tip.js') }}"></script> - <script src="{{ url_for_asset('graph.js') }}"></script> -{% endblock %} diff --git a/airflow/www/views.py b/airflow/www/views.py index fd4f8192ca..4227d13eb5 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -72,7 +72,7 @@ from pygments.formatters import HtmlFormatter from sqlalchemy import Date, and_, case, desc, func, inspect, or_, select, union_all from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, joinedload -from wtforms import SelectField, validators +from wtforms import validators import airflow from airflow import models, plugins_manager, settings @@ -136,7 +136,6 @@ from airflow.www.forms import ( DagRunEditForm, DateTimeForm, DateTimeWithNumRunsForm, - DateTimeWithNumRunsWithDagRunsForm, TaskInstanceEditForm, create_connection_form_class, ) @@ -3125,105 +3124,21 @@ class Airflow(AirflowBaseView): @action_logging @provide_session def graph(self, dag_id: str, session: Session = NEW_SESSION): - """Get DAG as Graph.""" + """Redirect to the replacement - grid + graph. Kept for backwards compatibility.""" dag = get_airflow_app().dag_bag.get_dag(dag_id, session=session) - dag_model = DagModel.get_dagmodel(dag_id, session=session) - if not dag: - flash(f'DAG "{dag_id}" seems to be missing.', "error") - return redirect(url_for("Airflow.index")) - - wwwutils.check_import_errors(dag.fileloc, session) - wwwutils.check_dag_warnings(dag.dag_id, session) - - root = request.args.get("root") - if root: - filter_upstream = request.args.get("filter_upstream") == "true" - filter_downstream = request.args.get("filter_downstream") == "true" - dag = dag.partial_subset( - task_ids_or_regex=root, include_upstream=filter_upstream, include_downstream=filter_downstream - ) - arrange = request.args.get("arrange", dag.orientation) - - nodes = task_group_to_dict(dag.task_group) - edges = dag_edges(dag) - dt_nr_dr_data = get_date_time_num_runs_dag_runs_form_data(request, session, dag) - - dt_nr_dr_data["arrange"] = arrange dttm = dt_nr_dr_data["dttm"] dag_run = dag.get_dagrun(execution_date=dttm) dag_run_id = dag_run.run_id if dag_run else None - class GraphForm(DateTimeWithNumRunsWithDagRunsForm): - """Graph Form class.""" - - arrange = SelectField( - "Layout", - choices=( - ("LR", "Left > Right"), - ("RL", "Right > Left"), - ("TB", "Top > Bottom"), - ("BT", "Bottom > Top"), - ), - ) - - form = GraphForm(data=dt_nr_dr_data) - form.execution_date.choices = dt_nr_dr_data["dr_choices"] - - task_instances = {} - for ti in dag.get_task_instances(dttm, dttm): - if ti.task_id not in task_instances: - task_instances[ti.task_id] = wwwutils.get_instance_with_map(ti, session) - # Need to add operator_name explicitly because it's not a column in task_instances model. - task_instances[ti.task_id]["operator_name"] = ti.operator_name - tasks = { - t.task_id: { - "dag_id": t.dag_id, - "task_type": t.task_type, - "operator_name": t.operator_name, - "extra_links": t.extra_links, - "is_mapped": isinstance(t, MappedOperator), - "trigger_rule": t.trigger_rule, - } - for t in dag.tasks + kwargs = { + **sanitize_args(request.args), + "dag_id": dag_id, + "tab": "graph", + "dag_run_id": dag_run_id, } - if not tasks: - flash("No tasks found", "error") - session.commit() - doc_md = wwwutils.wrapped_markdown(getattr(dag, "doc_md", None)) - - task_log_reader = TaskLogReader() - if task_log_reader.supports_external_link: - external_log_name = task_log_reader.log_handler.log_name - else: - external_log_name = None - - state_priority = ["no_status" if p is None else p for p in wwwutils.priority] - return self.render_template( - "airflow/graph.html", - dag=dag, - form=form, - dag_run_id=dag_run_id, - execution_date=dttm.isoformat(), - state_token=wwwutils.state_token(dt_nr_dr_data["dr_state"]), - doc_md=doc_md, - arrange=arrange, - operators=sorted( - {op.operator_name: op for op in dag.tasks}.values(), key=lambda x: x.operator_name - ), - root=root or "", - task_instances=task_instances, - tasks=tasks, - nodes=nodes, - edges=edges, - show_external_log_redirect=task_log_reader.supports_external_link, - external_log_name=external_log_name, - dag_run_state=dt_nr_dr_data["dr_state"], - dag_model=dag_model, - auto_refresh_interval=conf.getint("webserver", "auto_refresh_interval"), - state_priority=state_priority, - ) + return redirect(url_for("Airflow.grid", **kwargs)) @expose("/duration") @auth.has_access( diff --git a/airflow/www/webpack.config.js b/airflow/www/webpack.config.js index 98ef0bc3bf..2483faea32 100644 --- a/airflow/www/webpack.config.js +++ b/airflow/www/webpack.config.js @@ -65,7 +65,7 @@ const config = { dagDependencies: `${JS_DIR}/dag_dependencies.js`, dags: [`${CSS_DIR}/dags.css`, `${JS_DIR}/dags.js`], flash: `${CSS_DIR}/flash.css`, - graph: [`${CSS_DIR}/graph.css`, `${JS_DIR}/graph.js`], + graph: `${CSS_DIR}/graph.css`, loadingDots: `${CSS_DIR}/loading-dots.css`, main: [`${CSS_DIR}/main.css`, `${JS_DIR}/main.js`], materialIcons: `${CSS_DIR}/material-icons.css`, diff --git a/docs/apache-airflow/img/graph.png b/docs/apache-airflow/img/graph.png index 860fbff735..2c909c66c3 100644 Binary files a/docs/apache-airflow/img/graph.png and b/docs/apache-airflow/img/graph.png differ diff --git a/docs/apache-airflow/img/mapping-simple-graph.png b/docs/apache-airflow/img/mapping-simple-graph.png index 9e01d027ec..36ec7f9bc1 100644 Binary files a/docs/apache-airflow/img/mapping-simple-graph.png and b/docs/apache-airflow/img/mapping-simple-graph.png differ diff --git a/docs/apache-airflow/img/task_group.gif b/docs/apache-airflow/img/task_group.gif index ac4f6e943c..b7844f862d 100644 Binary files a/docs/apache-airflow/img/task_group.gif and b/docs/apache-airflow/img/task_group.gif differ diff --git a/tests/www/views/test_views_decorators.py b/tests/www/views/test_views_decorators.py index 13ede6273d..80eb588f29 100644 --- a/tests/www/views/test_views_decorators.py +++ b/tests/www/views/test_views_decorators.py @@ -100,36 +100,36 @@ def some_view_action_which_requires_dag_edit_access(*args) -> bool: def test_action_logging_get(session, admin_client): url = ( - f"dags/example_bash_operator/graph?" + f"dags/example_bash_operator/grid?" f"execution_date={urllib.parse.quote_plus(str(EXAMPLE_DAG_DEFAULT_DATE))}" ) resp = admin_client.get(url, follow_redirects=True) - check_content_in_response("runme_1", resp) + check_content_in_response("success", resp) # In mysql backend, this commit() is needed to write down the logs session.commit() _check_last_log( session, dag_id="example_bash_operator", - event="graph", + event="grid", execution_date=EXAMPLE_DAG_DEFAULT_DATE, ) def test_action_logging_get_legacy_view(session, admin_client): url = ( - f"graph?dag_id=example_bash_operator&" + f"tree?dag_id=example_bash_operator&" f"execution_date={urllib.parse.quote_plus(str(EXAMPLE_DAG_DEFAULT_DATE))}" ) resp = admin_client.get(url, follow_redirects=True) - check_content_in_response("runme_1", resp) + check_content_in_response("success", resp) # In mysql backend, this commit() is needed to write down the logs session.commit() _check_last_log( session, dag_id="example_bash_operator", - event="legacy_graph", + event="legacy_tree", execution_date=EXAMPLE_DAG_DEFAULT_DATE, ) diff --git a/tests/www/views/test_views_graph.py b/tests/www/views/test_views_graph.py deleted file mode 100644 index 3a49404d63..0000000000 --- a/tests/www/views/test_views_graph.py +++ /dev/null @@ -1,348 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -from datetime import timedelta -from urllib.parse import quote - -import pytest - -from airflow.configuration import conf -from airflow.models import DAG, DagRun -from airflow.models.baseoperator import BaseOperator -from airflow.utils import timezone -from airflow.utils.session import create_session, provide_session -from airflow.utils.state import State, TaskInstanceState - -DAG_ID = "dag_for_testing_dt_nr_dr_form" -DEFAULT_DATE = timezone.datetime(2017, 9, 1) -RUNS_DATA = [ - ("dag_run_for_testing_dt_nr_dr_form_4", timezone.datetime(2018, 4, 4)), - ("dag_run_for_testing_dt_nr_dr_form_3", timezone.datetime(2018, 3, 3)), - ("dag_run_for_testing_dt_nr_dr_form_2", timezone.datetime(2018, 2, 2)), - ("dag_run_for_testing_dt_nr_dr_form_1", timezone.datetime(2018, 1, 1)), -] -VERY_CLOSE_RUNS_DATE = timezone.datetime(2020, 1, 1, 0, 0, 0) - -ENDPOINTS = [ - "/graph?dag_id=dag_for_testing_dt_nr_dr_form", -] - - [email protected](scope="module") -def dag(app): - dag = DAG(DAG_ID, start_date=DEFAULT_DATE) - app.dag_bag.bag_dag(dag=dag, root_dag=dag) - return dag - - -@provide_session [email protected](scope="module") -def runs(dag, session): - dag_runs = [ - dag.create_dagrun( - run_id=run_id, - execution_date=execution_date, - data_interval=(execution_date, execution_date), - state=State.SUCCESS, - external_trigger=True, - ) - for run_id, execution_date, in RUNS_DATA - ] - yield dag_runs - for dag_run in dag_runs: - session.delete(dag_run) - - -def _assert_run_is_in_dropdown_not_selected(run, data): - exec_date = run.execution_date.isoformat() - assert f'<option value="{exec_date}">{run.run_id}</option>' in data - - -def _assert_run_is_selected(run, data): - exec_date = run.execution_date.isoformat() - assert f'<option selected value="{exec_date}">{run.run_id}</option>' in data - - -def _assert_base_date(base_date, data): - assert f'name="base_date" required type="text" value="{base_date.isoformat()}"' in data - - -def _assert_base_date_and_num_runs(base_date, num, data): - assert f'name="base_date" value="{base_date}"' not in data - assert f'<option selected="" value="{num}">{num}</option>' not in data - - -def _assert_run_is_not_in_dropdown(run, data): - assert run.execution_date.isoformat() not in data - assert run.run_id not in data - - [email protected]("endpoint", ENDPOINTS) -def test_with_default_parameters(admin_client, runs, endpoint): - """ - Tests view with no URL parameter. - Should show all dag runs in the drop down. - Should select the latest dag run. - Should set base date to current date (not asserted) - """ - response = admin_client.get( - endpoint, data={"username": "test", "password": "test"}, follow_redirects=True - ) - assert response.status_code == 200 - - data = response.data.decode() - assert '<label class="sr-only" for="base_date">Base date</label>' in data - assert '<label class="sr-only" for="num_runs">Number of runs</label>' in data - - _assert_run_is_selected(runs[0], data) - _assert_run_is_in_dropdown_not_selected(runs[1], data) - _assert_run_is_in_dropdown_not_selected(runs[2], data) - _assert_run_is_in_dropdown_not_selected(runs[3], data) - - [email protected]("endpoint", ENDPOINTS) -def test_with_execution_date_parameter_only(admin_client, runs, endpoint): - """ - Tests view with execution_date URL parameter. - Scenario: click link from dag runs view. - Should only show dag runs older than execution_date in the drop down. - Should select the particular dag run. - Should set base date to execution date. - """ - response = admin_client.get( - f"{endpoint}&execution_date={runs[1].execution_date.isoformat()}", - data={"username": "test", "password": "test"}, - follow_redirects=True, - ) - assert response.status_code == 200 - - data = response.data.decode() - _assert_base_date_and_num_runs( - runs[1].execution_date, - conf.getint("webserver", "default_dag_run_display_number"), - data, - ) - _assert_run_is_not_in_dropdown(runs[0], data) - _assert_run_is_selected(runs[1], data) - _assert_run_is_in_dropdown_not_selected(runs[2], data) - _assert_run_is_in_dropdown_not_selected(runs[3], data) - - [email protected]("endpoint", ENDPOINTS) -def test_with_base_date_and_num_runs_parameters_only(admin_client, runs, endpoint): - """ - Tests view with base_date and num_runs URL parameters. - Should only show dag runs older than base_date in the drop down, - limited to num_runs. - Should select the latest dag run. - Should set base date and num runs to submitted values. - """ - response = admin_client.get( - f"{endpoint}&base_date={runs[1].execution_date.isoformat()}&num_runs=2", - data={"username": "test", "password": "test"}, - follow_redirects=True, - ) - assert response.status_code == 200 - - data = response.data.decode() - _assert_base_date_and_num_runs(runs[1].execution_date, 2, data) - _assert_run_is_not_in_dropdown(runs[0], data) - _assert_run_is_selected(runs[1], data) - _assert_run_is_in_dropdown_not_selected(runs[2], data) - _assert_run_is_not_in_dropdown(runs[3], data) - - [email protected]("endpoint", ENDPOINTS) -def test_with_base_date_and_num_runs_and_execution_date_outside(admin_client, runs, endpoint): - """ - Tests view with base_date and num_runs and execution-date URL parameters. - Scenario: change the base date and num runs and press "Go", - the selected execution date is outside the new range. - Should only show dag runs older than base_date in the drop down. - Should select the latest dag run within the range. - Should set base date and num runs to submitted values. - """ - base_date = runs[1].execution_date.isoformat() - exec_date = runs[0].execution_date.isoformat() - response = admin_client.get( - f"{endpoint}&base_date={base_date}&num_runs=42&execution_date={exec_date}", - data={"username": "test", "password": "test"}, - follow_redirects=True, - ) - assert response.status_code == 200 - - data = response.data.decode() - _assert_base_date_and_num_runs(runs[1].execution_date, 42, data) - _assert_run_is_not_in_dropdown(runs[0], data) - _assert_run_is_selected(runs[1], data) - _assert_run_is_in_dropdown_not_selected(runs[2], data) - _assert_run_is_in_dropdown_not_selected(runs[3], data) - - [email protected]("endpoint", ENDPOINTS) -def test_with_base_date_and_num_runs_and_execution_date_within(admin_client, runs, endpoint): - """ - Tests view with base_date and num_runs and execution-date URL parameters. - Scenario: change the base date and num runs and press "Go", - the selected execution date is within the new range. - Should only show dag runs older than base_date in the drop down. - Should select the dag run with the execution date. - Should set base date and num runs to submitted values. - """ - base_date = runs[2].execution_date.isoformat() - exec_date = runs[3].execution_date.isoformat() - response = admin_client.get( - f"{endpoint}&base_date={base_date}&num_runs=5&execution_date={exec_date}", - data={"username": "test", "password": "test"}, - follow_redirects=True, - ) - assert response.status_code == 200 - - data = response.data.decode() - _assert_base_date_and_num_runs(runs[2].execution_date, 5, data) - _assert_run_is_not_in_dropdown(runs[0], data) - _assert_run_is_not_in_dropdown(runs[1], data) - _assert_run_is_in_dropdown_not_selected(runs[2], data) - _assert_run_is_selected(runs[3], data) - - -@provide_session [email protected] -def very_close_dagruns(dag, session): - dag_runs = [] - for idx, (run_id, _) in enumerate(RUNS_DATA): - execution_date = VERY_CLOSE_RUNS_DATE.replace(microsecond=idx) - dag_runs.append( - dag.create_dagrun( - run_id=run_id + "_close", - execution_date=execution_date, - data_interval=(execution_date, execution_date), - state=State.SUCCESS, - external_trigger=True, - ) - ) - yield dag_runs - for dag_run in dag_runs: - session.delete(dag_run) - session.commit() - - [email protected]("endpoint", ENDPOINTS) -def test_rounds_base_date_but_queries_with_execution_date(admin_client, very_close_dagruns, endpoint): - exec_date = quote(very_close_dagruns[1].execution_date.isoformat()) - response = admin_client.get( - f"{endpoint}&num_runs=2&execution_date={exec_date}", - data={"username": "test", "password": "test"}, - follow_redirects=True, - ) - assert response.status_code == 200 - - data = response.data.decode() - _assert_base_date(VERY_CLOSE_RUNS_DATE + timedelta(seconds=1), data) - _assert_run_is_in_dropdown_not_selected(very_close_dagruns[0], data) - _assert_run_is_selected(very_close_dagruns[1], data) - _assert_run_is_not_in_dropdown(very_close_dagruns[2], data) - _assert_run_is_not_in_dropdown(very_close_dagruns[3], data) - - [email protected]("endpoint", ENDPOINTS) -def test_uses_execution_date_on_filter_application_if_base_date_hasnt_changed( - admin_client, very_close_dagruns, endpoint -): - base_date = quote((VERY_CLOSE_RUNS_DATE + timedelta(seconds=1)).isoformat()) - exec_date = quote(very_close_dagruns[1].execution_date.isoformat()) - response = admin_client.get( - f"{endpoint}&base_date={base_date}&num_runs=2&execution_date={exec_date}", - data={"username": "test", "password": "test"}, - follow_redirects=True, - ) - assert response.status_code == 200 - - data = response.data.decode() - _assert_base_date(VERY_CLOSE_RUNS_DATE + timedelta(seconds=1), data) - _assert_run_is_in_dropdown_not_selected(very_close_dagruns[0], data) - _assert_run_is_selected(very_close_dagruns[1], data) - _assert_run_is_not_in_dropdown(very_close_dagruns[2], data) - _assert_run_is_not_in_dropdown(very_close_dagruns[3], data) - - [email protected]("endpoint", ENDPOINTS) -def test_uses_base_date_if_changed_away_from_execution_date(admin_client, very_close_dagruns, endpoint): - base_date = quote((VERY_CLOSE_RUNS_DATE + timedelta(seconds=2)).isoformat()) - exec_date = quote(very_close_dagruns[1].execution_date.isoformat()) - response = admin_client.get( - f"{endpoint}&base_date={base_date}&num_runs=2&execution_date={exec_date}", - data={"username": "test", "password": "test"}, - follow_redirects=True, - ) - assert response.status_code == 200 - - data = response.data.decode() - _assert_base_date(VERY_CLOSE_RUNS_DATE + timedelta(seconds=2), data) - _assert_run_is_not_in_dropdown(very_close_dagruns[0], data) - _assert_run_is_not_in_dropdown(very_close_dagruns[1], data) - _assert_run_is_in_dropdown_not_selected(very_close_dagruns[2], data) - _assert_run_is_selected(very_close_dagruns[3], data) - - [email protected]("endpoint", ENDPOINTS) -def test_view_works_with_deleted_tasks(request, admin_client, app, endpoint): - task_to_state = { - "existing-task": TaskInstanceState.SUCCESS, - "deleted-task-success": TaskInstanceState.SUCCESS, - "deleted-task-failed": TaskInstanceState.FAILED, - } - dag = DAG(DAG_ID, start_date=DEFAULT_DATE) - for task_id in task_to_state.keys(): - BaseOperator(task_id=task_id, dag=dag) - - execution_date = timezone.datetime(2022, 3, 14) - dag_run_id = "test-deleted-tasks-dag-run" - with create_session() as session: - dag_run = dag.create_dagrun( - run_id=dag_run_id, - execution_date=execution_date, - data_interval=(execution_date, execution_date + timedelta(minutes=5)), - state=State.SUCCESS, - external_trigger=True, - session=session, - ) - for ti in dag_run.task_instances: - ti.refresh_from_task(dag.get_task(ti.task_id)) - ti.state = task_to_state[ti.task_id] - ti.start_date = execution_date - ti.end_date = execution_date + timedelta(minutes=5) - session.merge(ti) - - def cleanup_database(): - with create_session() as session: - session.query(DagRun).filter_by(run_id=dag_run_id).delete() - - request.addfinalizer(cleanup_database) - - dag = DAG(DAG_ID, start_date=DEFAULT_DATE) - BaseOperator(task_id="existing-task", dag=dag) - app.dag_bag.bag_dag(dag=dag, root_dag=dag) - - response = admin_client.get( - f"{endpoint}&execution_date={execution_date.isoformat()}", - data={"username": "test", "password": "test"}, - follow_redirects=True, - ) - assert response.status_code == 200 diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 32df99c047..3685f2323c 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -174,14 +174,14 @@ def client_ti_without_dag_edit(app): id="dag-details-subdag", ), pytest.param( - "graph?dag_id=example_bash_operator", + "object/graph_data?dag_id=example_bash_operator", ["runme_1"], - id="graph-url-param", + id="graph-data", ), pytest.param( - "dags/example_bash_operator/graph", - ["runme_1"], - id="graph", + "object/graph_data?dag_id=example_subdag_operator.section-1", + ["section-1-task-1"], + id="graph-data-subdag", ), pytest.param( "object/grid_data?dag_id=example_bash_operator", @@ -370,7 +370,7 @@ def test_tree_trigger_origin_tree_view(app, admin_client): check_content_in_response(href, resp) -def test_graph_trigger_origin_graph_view(app, admin_client): +def test_graph_trigger_origin_grid_view(app, admin_client): app.dag_bag.get_dag("test_tree_view").create_dagrun( run_type=DagRunType.SCHEDULED, execution_date=DEFAULT_DATE, @@ -381,7 +381,23 @@ def test_graph_trigger_origin_graph_view(app, admin_client): url = "/dags/test_tree_view/graph" resp = admin_client.get(url, follow_redirects=True) - params = {"origin": "/dags/test_tree_view/graph"} + params = {"origin": "/dags/test_tree_view/grid?tab=graph"} + href = f"/dags/test_tree_view/trigger?{html.escape(urllib.parse.urlencode(params))}" + check_content_in_response(href, resp) + + +def test_gantt_trigger_origin_grid_view(app, admin_client): + app.dag_bag.get_dag("test_tree_view").create_dagrun( + run_type=DagRunType.SCHEDULED, + execution_date=DEFAULT_DATE, + data_interval=(DEFAULT_DATE, DEFAULT_DATE), + start_date=timezone.utcnow(), + state=State.RUNNING, + ) + + url = "/dags/test_tree_view/gantt" + resp = admin_client.get(url, follow_redirects=True) + params = {"origin": "/dags/test_tree_view/grid?tab=gantt"} href = f"/dags/test_tree_view/trigger?{html.escape(urllib.parse.urlencode(params))}" check_content_in_response(href, resp) @@ -390,7 +406,10 @@ def test_graph_view_without_dag_permission(app, one_dag_perm_user_client): url = "/dags/example_bash_operator/graph" resp = one_dag_perm_user_client.get(url, follow_redirects=True) assert resp.status_code == 200 - assert resp.request.url == "http://localhost/dags/example_bash_operator/graph"; + assert ( + resp.request.url + == "http://localhost/dags/example_bash_operator/grid?tab=graph&dag_run_id=TEST_DAGRUN"; + ) check_content_in_response("example_bash_operator", resp) url = "/dags/example_xcom/graph"
