This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 02d94c5f0e Migration of all DAG details to existing grid view dag
details panel (#31690)
02d94c5f0e is described below
commit 02d94c5f0e2ca4571cdfbde2a8624dd4569876c7
Author: Akash Sharma <[email protected]>
AuthorDate: Tue Jun 6 22:03:27 2023 +0530
Migration of all DAG details to existing grid view dag details panel
(#31690)
* dag details added in grid view
* dag details data integrated and all links added
* deleted old dag_details and all refs
* reverted view.py and bug resolution on object parsing
* resolved bug while combining dag and dagDetails
* failing tests fixed
* Update airflow/www/static/js/dag/details/Dag.tsx
Co-authored-by: Brent Bovenzi <[email protected]>
* Update airflow/www/static/js/components/ViewScheduleInterval.tsx
Co-authored-by: Brent Bovenzi <[email protected]>
* code clean and static checks fixed
---------
Co-authored-by: Brent Bovenzi <[email protected]>
---
airflow/www/static/js/api/index.ts | 2 +
airflow/www/static/js/api/useDagDetails.ts | 53 ++++++++
.../static/js/components/ViewScheduleInterval.tsx | 61 +++++++++
airflow/www/static/js/dag/details/Dag.tsx | 148 +++++++++++++++++++--
airflow/www/static/js/utils/index.ts | 9 ++
airflow/www/templates/airflow/dag.html | 3 +-
airflow/www/templates/airflow/dags.html | 4 -
7 files changed, 267 insertions(+), 13 deletions(-)
diff --git a/airflow/www/static/js/api/index.ts
b/airflow/www/static/js/api/index.ts
index cd7b9dd5af..94ec52e863 100644
--- a/airflow/www/static/js/api/index.ts
+++ b/airflow/www/static/js/api/index.ts
@@ -42,6 +42,7 @@ import useUpstreamDatasetEvents from
"./useUpstreamDatasetEvents";
import useTaskInstance from "./useTaskInstance";
import useDag from "./useDag";
import useDagCode from "./useDagCode";
+import useDagDetails from "./useDagDetails";
import useHealth from "./useHealth";
import usePools from "./usePools";
import useDags from "./useDags";
@@ -59,6 +60,7 @@ export {
useClearTask,
useDag,
useDagCode,
+ useDagDetails,
useDagRuns,
useDags,
useDataset,
diff --git a/airflow/www/static/js/api/useDagDetails.ts
b/airflow/www/static/js/api/useDagDetails.ts
new file mode 100644
index 0000000000..16c8e4397c
--- /dev/null
+++ b/airflow/www/static/js/api/useDagDetails.ts
@@ -0,0 +1,53 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import axios, { AxiosResponse } from "axios";
+import { useQuery } from "react-query";
+
+import { getMetaValue } from "src/utils";
+import type { API } from "src/types";
+import type { DAG, DAGDetail } from "src/types/api-generated";
+import useDag from "./useDag";
+
+const dagDetailsApiUrl = getMetaValue("dag_details_api");
+
+const combineResults = (
+ dagData: DAG,
+ dagDetailsData: DAGDetail
+): Omit<DAGDetail, "defaultView"> => ({ ...dagData, ...dagDetailsData });
+
+const useDagDetails = () => {
+ const { data: dagData } = useDag();
+ const dagDetailsResult = useQuery(
+ ["dagDetailsQuery"],
+ () => axios.get<AxiosResponse, API.DAGDetail>(dagDetailsApiUrl),
+ {
+ enabled: !!dagData,
+ }
+ );
+ return {
+ ...dagDetailsResult,
+ data: combineResults(
+ dagData || {},
+ dagDetailsResult.data ? dagDetailsResult.data : {}
+ ),
+ };
+};
+
+export default useDagDetails;
diff --git a/airflow/www/static/js/components/ViewScheduleInterval.tsx
b/airflow/www/static/js/components/ViewScheduleInterval.tsx
new file mode 100644
index 0000000000..68bc5cf1f5
--- /dev/null
+++ b/airflow/www/static/js/components/ViewScheduleInterval.tsx
@@ -0,0 +1,61 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from "react";
+import { Flex, Text } from "@chakra-ui/react";
+
+interface Props {
+ data: Record<string, number | null>;
+}
+
+const ViewScheduleInterval = ({ data }: Props) => {
+ const genericTimeDeltaUnits = [
+ "days",
+ "day",
+ "seconds",
+ "microseconds",
+ "years",
+ "year",
+ "months",
+ "month",
+ "leapdays",
+ "hours",
+ "hour",
+ "minutes",
+ "minute",
+ "second",
+ "microsecond",
+ ];
+
+ return (
+ <Flex flexWrap="wrap">
+ {!!data &&
+ genericTimeDeltaUnits.map(
+ (unit) =>
+ !!data[unit] && (
+ <Text mr={1} key={unit}>
+ {data[unit]} {unit}
+ </Text>
+ )
+ )}
+ </Flex>
+ );
+};
+
+export default ViewScheduleInterval;
diff --git a/airflow/www/static/js/dag/details/Dag.tsx
b/airflow/www/static/js/dag/details/Dag.tsx
index 098eb47dbd..836c8960b3 100644
--- a/airflow/www/static/js/dag/details/Dag.tsx
+++ b/airflow/www/static/js/dag/details/Dag.tsx
@@ -29,23 +29,32 @@ import {
Heading,
Text,
Box,
+ Badge,
+ Code,
} from "@chakra-ui/react";
-import { mean } from "lodash";
+import { mean, omit } from "lodash";
import { getDuration, formatDuration } from "src/datetime_utils";
import {
+ appendSearchParams,
finalStatesMap,
getMetaValue,
getTaskSummary,
+ toSentenceCase,
useOffsetTop,
} from "src/utils";
-import { useGridData } from "src/api";
+import { useGridData, useDagDetails } from "src/api";
import Time from "src/components/Time";
+import ViewScheduleInterval from "src/components/ViewScheduleInterval";
import type { TaskState } from "src/types";
+import type { DAG, DAGDetail } from "src/types/api-generated";
+import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper";
import { SimpleStatus } from "../StatusBox";
-const dagDetailsUrl = getMetaValue("dag_details_url");
+const dagId = getMetaValue("dag_id");
+const tagIndexUrl = getMetaValue("tag_index_url");
+const taskInstancesUrl = getMetaValue("task_instances_list_url");
const Dag = () => {
const {
@@ -54,9 +63,32 @@ const Dag = () => {
const detailsRef = useRef<HTMLDivElement>(null);
const offsetTop = useOffsetTop(detailsRef);
+ const { data: dagDetailsData, isLoading: isLoadingDagDetails } =
+ useDagDetails();
+
+ // fields to exclude from "dagDetailsData" since handled seprately or not
required
+ const dagDataExcludeFields = [
+ "defaultView",
+ "fileToken",
+ "scheduleInterval",
+ "tags",
+ "owners",
+ "params",
+ ];
+
+ const listParams = new URLSearchParamsWrapper({
+ _flt_3_dag_id: dagId,
+ });
+
+ const getRedirectUri = (state: string): string => {
+ listParams.set("_flt_3_state", state);
+ return appendSearchParams(taskInstancesUrl, listParams);
+ };
+
const taskSummary = getTaskSummary({ task: groups });
const numMap = finalStatesMap();
const durations: number[] = [];
+
dagRuns.forEach((dagRun) => {
durations.push(getDuration(dagRun.startDate, dagRun.endDate));
const stateKey = dagRun.state == null ? "no_status" : dagRun.state;
@@ -72,8 +104,15 @@ const Dag = () => {
<Tr key={val}>
<Td>
<Flex alignItems="center">
- <SimpleStatus state={val as TaskState} mr={2} />
- <Text>Total {val}</Text>
+ <SimpleStatus state={val as TaskState} />
+ <Link
+ href={getRedirectUri(val)}
+ title={`View all ${val} DAGS`}
+ color="blue"
+ ml="5px"
+ >
+ Total {val}
+ </Link>
</Flex>
</Td>
<Td>{key}</Td>
@@ -89,6 +128,28 @@ const Dag = () => {
const firstStart = dagRuns[0]?.startDate;
const lastStart = dagRuns[dagRuns.length - 1]?.startDate;
+ // parse value for each key if date or not
+ const parseStringData = (value: string) =>
+ Number.isNaN(Date.parse(value)) ? value : <Time dateTime={value} />;
+
+ // render dag and dag_details data
+ const renderDagDetailsData = (
+ data: DAG | DAGDetail,
+ excludekeys: Array<string>
+ ) => (
+ <>
+ {Object.entries(data).map(
+ ([key, value]) =>
+ !excludekeys.includes(key) && (
+ <Tr key={key}>
+ <Td>{toSentenceCase(key)}</Td>
+ <Td>{parseStringData(String(value))}</Td>
+ </Tr>
+ )
+ )}
+ </>
+ );
+
return (
<Box
height="100%"
@@ -96,9 +157,6 @@ const Dag = () => {
ref={detailsRef}
overflowY="auto"
>
- <Button as={Link} variant="ghost" colorScheme="blue"
href={dagDetailsUrl}>
- More Details
- </Button>
<Table variant="striped">
<Tbody>
{durations.length > 0 && (
@@ -169,6 +227,80 @@ const Dag = () => {
<Td>{value}</Td>
</Tr>
))}
+ {!isLoadingDagDetails && !!dagDetailsData && (
+ <>
+ <Tr borderBottomWidth={2} borderBottomColor="gray.300">
+ <Td>
+ <Heading size="sm">DAG Details</Heading>
+ </Td>
+ <Td />
+ </Tr>
+ {renderDagDetailsData(dagDetailsData, dagDataExcludeFields)}
+ <Tr>
+ <Td>Owners</Td>
+ <Td>
+ <Flex flexWrap="wrap">
+ {dagDetailsData.owners?.map((owner) => (
+ <Badge key={owner} colorScheme="blue">
+ {owner}
+ </Badge>
+ ))}
+ </Flex>
+ </Td>
+ </Tr>
+ <Tr>
+ <Td>Tags</Td>
+ <Td>
+ {!!dagDetailsData.tags && dagDetailsData.tags?.length > 0 ? (
+ <Flex flexWrap="wrap">
+ {dagDetailsData.tags?.map((tag) => (
+ <Button
+ key={tag.name}
+ as={Link}
+ colorScheme="teal"
+ size="xs"
+ href={tagIndexUrl.replace(
+ "_TAG_NAME_",
+ tag?.name || ""
+ )}
+ mr={3}
+ >
+ {tag.name}
+ </Button>
+ ))}
+ </Flex>
+ ) : (
+ "No tags"
+ )}
+ </Td>
+ </Tr>
+ <Tr>
+ <Td>Schedule interval</Td>
+ <Td>
+ {dagDetailsData.scheduleInterval?.type ===
+ "CronExpression" ? (
+ <Text>{dagDetailsData.scheduleInterval?.value}</Text>
+ ) : (
+ // for TimeDelta and RelativeDelta
+ <ViewScheduleInterval
+ data={omit(dagDetailsData.scheduleInterval, [
+ "type",
+ "value",
+ ])}
+ />
+ )}
+ </Td>
+ </Tr>
+ <Tr>
+ <Td>Params</Td>
+ <Td>
+ <Code width="100%">
+ <pre>{JSON.stringify(dagDetailsData.params, null, 2)}</pre>
+ </Code>
+ </Td>
+ </Tr>
+ </>
+ )}
</Tbody>
</Table>
</Box>
diff --git a/airflow/www/static/js/utils/index.ts
b/airflow/www/static/js/utils/index.ts
index 7526adfff7..37dc5cb022 100644
--- a/airflow/www/static/js/utils/index.ts
+++ b/airflow/www/static/js/utils/index.ts
@@ -177,6 +177,14 @@ const getStatusBackgroundColor = (color: string, hasNote:
boolean) =>
? `linear-gradient(-135deg, ${Color(color).hex()}60 5px, ${color} 0);`
: color;
+const toSentenceCase = (camelCase: string): string => {
+ if (camelCase) {
+ const result = camelCase.replace(/([A-Z])/g, " $1");
+ return result[0].toUpperCase() + result.substring(1).toLowerCase();
+ }
+ return "";
+};
+
export {
hoverDelay,
finalStatesMap,
@@ -188,4 +196,5 @@ export {
getDagRunLabel,
getStatusBackgroundColor,
useOffsetTop,
+ toSentenceCase,
};
diff --git a/airflow/www/templates/airflow/dag.html
b/airflow/www/templates/airflow/dag.html
index 3a2c448f20..66531c607e 100644
--- a/airflow/www/templates/airflow/dag.html
+++ b/airflow/www/templates/airflow/dag.html
@@ -60,7 +60,6 @@
<meta name="grid_url" content="{{ url_for('Airflow.grid', dag_id=dag.dag_id)
}}">
<meta name="datasets_url" content="{{ url_for('Airflow.datasets') }}">
<meta name="grid_url_no_root" content="{{ url_for('Airflow.grid',
dag_id=dag.dag_id, num_runs=num_runs_arg, base_date=base_date_arg) }}">
- <meta name="dag_details_url" content="{{ url_for('Airflow.dag_details',
dag_id=dag.dag_id) }}">
<meta name="graph_url" content="{{ url_for('Airflow.graph',
dag_id=dag.dag_id, root=root) }}">
<meta name="task_url" content="{{ url_for('Airflow.task', dag_id=dag.dag_id)
}}">
<meta name="log_url" content="{{ url_for('Airflow.log', dag_id=dag.dag_id)
}}">
@@ -68,6 +67,7 @@
<meta name="rendered_templates_url" content="{{
url_for('Airflow.rendered_templates', dag_id=dag.dag_id) }}">
<meta name="rendered_k8s_url" content="{{ url_for('Airflow.rendered_k8s',
dag_id=dag.dag_id) }}">
<meta name="task_instances_list_url" content="{{
url_for('TaskInstanceModelView.list') }}">
+ <meta name="tag_index_url" content="{{ url_for('Airflow.index',
tags='_TAG_NAME_') }}">
<meta name="mapped_instances_api" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_task_instance_endpoint_get_mapped_task_instances',
dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_', task_id='_TASK_ID_') }}">
<meta name="task_log_api" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_log_endpoint_get_log',
dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_', task_id='_TASK_ID_',
task_try_number='-1') }}">
<meta name="upstream_dataset_events_api" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_dag_run_endpoint_get_upstream_dataset_events',
dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_') }}">
@@ -77,6 +77,7 @@
<meta name="set_dag_run_note" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_dag_run_endpoint_set_dag_run_note',
dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_') }}">
<meta name="dag_api" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_dag_endpoint_get_dag',
dag_id=dag.dag_id) }}">
<meta name="dag_source_api" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_dag_source_endpoint_get_dag_source',
file_token='_FILE_TOKEN_') }}">
+ <meta name="dag_details_api" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_dag_endpoint_get_dag_details',
dag_id=dag.dag_id) }}">
<!-- End Urls -->
<meta name="is_paused" content="{{ dag_is_paused }}">
diff --git a/airflow/www/templates/airflow/dags.html
b/airflow/www/templates/airflow/dags.html
index 393d511f19..80b37f6a42 100644
--- a/airflow/www/templates/airflow/dags.html
+++ b/airflow/www/templates/airflow/dags.html
@@ -387,10 +387,6 @@
<span class="material-icons"
aria-hidden="true">code</span>
Code
</a>
- <a href="{{ url_for('Airflow.dag_details',
dag_id=dag.dag_id) }}" class="dags-table-more__link">
- <span class="material-icons"
aria-hidden="true">details</span>
- Details
- </a>
<a href="{{ url_for('Airflow.gantt', dag_id=dag.dag_id)
}}" class="dags-table-more__link">
<span class="material-icons"
aria-hidden="true">vertical_distribute</span>
Gantt
