This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 1074b8eed6 Add Rendered k8s pod spec tab to ti details view (#39141)
1074b8eed6 is described below
commit 1074b8eed680af9668f11c63cf28e72db5470fde
Author: Brent Bovenzi <[email protected]>
AuthorDate: Thu May 2 11:21:45 2024 -0400
Add Rendered k8s pod spec tab to ti details view (#39141)
* Add Rendered k8s pod spec tab to ti details view
* Render yaml instead of json
* Fix rebase mistake
---
airflow/www/package.json | 4 +-
airflow/www/static/js/api/index.ts | 2 +
airflow/www/static/js/api/useRenderedK8s.ts | 43 +++++++++++++++++
airflow/www/static/js/dag/details/index.tsx | 23 ++++++++-
.../www/static/js/dag/details/taskInstance/Nav.tsx | 6 ---
.../js/dag/details/taskInstance/RenderedK8s.tsx | 55 ++++++++++++++++++++++
airflow/www/templates/airflow/dag.html | 2 +-
airflow/www/views.py | 44 ++++++++++++++++-
airflow/www/yarn.lock | 31 ++++++++++--
9 files changed, 196 insertions(+), 14 deletions(-)
diff --git a/airflow/www/package.json b/airflow/www/package.json
index 22b6f882d3..ef24c53a78 100644
--- a/airflow/www/package.json
+++ b/airflow/www/package.json
@@ -51,6 +51,7 @@
"@testing-library/jest-dom": "^5.16.0",
"@testing-library/react": "^13.0.0",
"@types/color": "^3.0.3",
+ "@types/json-to-pretty-yaml": "^1.2.1",
"@types/react": "^18.0.12",
"@types/react-dom": "^18.0.5",
"@types/react-syntax-highlighter": "^15.5.6",
@@ -126,11 +127,12 @@
"framer-motion": "^6.0.0",
"jquery": ">=3.5.0",
"jshint": "^2.13.4",
+ "json-to-pretty-yaml": "^1.2.2",
"lodash": "^4.17.21",
"moment-timezone": "^0.5.43",
"react": "^18.0.0",
"react-dom": "^18.0.0",
- "react-icons": "^4.9.0",
+ "react-icons": "^5.1.0",
"react-json-view": "^1.21.3",
"react-markdown": "^8.0.4",
"react-query": "^3.39.1",
diff --git a/airflow/www/static/js/api/index.ts
b/airflow/www/static/js/api/index.ts
index 4ba363a75c..c9bc4ed4c8 100644
--- a/airflow/www/static/js/api/index.ts
+++ b/airflow/www/static/js/api/index.ts
@@ -53,6 +53,7 @@ import { useTaskXcomEntry, useTaskXcomCollection } from
"./useTaskXcom";
import useEventLogs from "./useEventLogs";
import useCalendarData from "./useCalendarData";
import useCreateDatasetEvent from "./useCreateDatasetEvent";
+import useRenderedK8s from "./useRenderedK8s";
axios.interceptors.request.use((config) => {
config.paramsSerializer = {
@@ -102,4 +103,5 @@ export {
useEventLogs,
useCalendarData,
useCreateDatasetEvent,
+ useRenderedK8s,
};
diff --git a/airflow/www/static/js/api/useRenderedK8s.ts
b/airflow/www/static/js/api/useRenderedK8s.ts
new file mode 100644
index 0000000000..1b1828e569
--- /dev/null
+++ b/airflow/www/static/js/api/useRenderedK8s.ts
@@ -0,0 +1,43 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import axios, { AxiosResponse } from "axios";
+import { useQuery } from "react-query";
+
+import { getMetaValue } from "src/utils";
+
+const url = getMetaValue("rendered_k8s_data_url");
+
+const useRenderedK8s = (
+ runId: string | null,
+ taskId: string | null,
+ mapIndex?: number
+) =>
+ useQuery(
+ ["rendered_k8s", runId, taskId, mapIndex],
+ async () =>
+ axios.get<AxiosResponse, any>(url, {
+ params: { run_id: runId, task_id: taskId, map_index: mapIndex },
+ }),
+ {
+ enabled: !!runId && !!taskId,
+ }
+ );
+
+export default useRenderedK8s;
diff --git a/airflow/www/static/js/dag/details/index.tsx
b/airflow/www/static/js/dag/details/index.tsx
index 954792f968..33d2b68ef4 100644
--- a/airflow/www/static/js/dag/details/index.tsx
+++ b/airflow/www/static/js/dag/details/index.tsx
@@ -45,7 +45,7 @@ import {
MdPlagiarism,
MdEvent,
} from "react-icons/md";
-import { BiBracket } from "react-icons/bi";
+import { BiBracket, BiLogoKubernetes } from "react-icons/bi";
import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper";
import Header from "./Header";
@@ -68,6 +68,7 @@ import TaskDetails from "./task";
import AuditLog from "./AuditLog";
import RunDuration from "./dag/RunDuration";
import Calendar from "./dag/Calendar";
+import RenderedK8s from "./taskInstance/RenderedK8s";
const dagId = getMetaValue("dag_id")!;
@@ -79,6 +80,8 @@ interface Props {
ganttScrollRef: React.RefObject<HTMLDivElement>;
}
+const isK8sExecutor = getMetaValue("k8s_or_k8scelery_executor") === "True";
+
const tabToIndex = (tab?: string) => {
switch (tab) {
case "graph":
@@ -96,6 +99,8 @@ const tabToIndex = (tab?: string) => {
case "xcom":
case "calendar":
return 6;
+ case "rendered_k8s":
+ return 7;
case "details":
default:
return 0;
@@ -135,6 +140,9 @@ const indexToTab = (
if (!runId && !taskId) return "calendar";
if (isTaskInstance) return "xcom";
return undefined;
+ case 7:
+ if (isTaskInstance && isK8sExecutor) return "rendered_k8s";
+ return undefined;
default:
return undefined;
}
@@ -360,6 +368,14 @@ const Details = ({
</Text>
</Tab>
)}
+ {isTaskInstance && isK8sExecutor && (
+ <Tab>
+ <BiLogoKubernetes size={16} />
+ <Text as="strong" ml={1}>
+ K8s Pod Spec
+ </Text>
+ </Tab>
+ )}
{/* Match the styling of a tab but its actually a button */}
{!!taskId && !!runId && (
<Button
@@ -484,6 +500,11 @@ const Details = ({
/>
</TabPanel>
)}
+ {isTaskInstance && isK8sExecutor && (
+ <TabPanel height="100%">
+ <RenderedK8s />
+ </TabPanel>
+ )}
</TabPanels>
</Tabs>
</Flex>
diff --git a/airflow/www/static/js/dag/details/taskInstance/Nav.tsx
b/airflow/www/static/js/dag/details/taskInstance/Nav.tsx
index 88560d5dac..e24088085a 100644
--- a/airflow/www/static/js/dag/details/taskInstance/Nav.tsx
+++ b/airflow/www/static/js/dag/details/taskInstance/Nav.tsx
@@ -26,9 +26,7 @@ import type { Task } from "src/types";
import URLSearchParamsWrapper from "src/utils/URLSearchParamWrapper";
const dagId = getMetaValue("dag_id");
-const isK8sExecutor = getMetaValue("k8s_or_k8scelery_executor") === "True";
const taskInstancesUrl = getMetaValue("task_instances_list_url");
-const renderedK8sUrl = getMetaValue("rendered_k8s_url");
const taskUrl = getMetaValue("task_url");
const gridUrl = getMetaValue("grid_url");
@@ -49,7 +47,6 @@ const Nav = forwardRef<HTMLDivElement, Props>(
map_index: mapIndex ?? -1,
});
const detailsLink = `${taskUrl}&${params}`;
- const k8sLink = `${renderedK8sUrl}&${params}`;
const listParams = new URLSearchParamsWrapper({
_flt_3_dag_id: dagId,
_flt_3_task_id: taskId,
@@ -77,9 +74,6 @@ const Nav = forwardRef<HTMLDivElement, Props>(
{(!isMapped || mapIndex !== undefined) && (
<>
<LinkButton href={detailsLink}>More Details</LinkButton>
- {isK8sExecutor && (
- <LinkButton href={k8sLink}>K8s Pod Spec</LinkButton>
- )}
{isSubDag && (
<LinkButton href={subDagLink}>Zoom into SubDag</LinkButton>
)}
diff --git a/airflow/www/static/js/dag/details/taskInstance/RenderedK8s.tsx
b/airflow/www/static/js/dag/details/taskInstance/RenderedK8s.tsx
new file mode 100644
index 0000000000..459d191edd
--- /dev/null
+++ b/airflow/www/static/js/dag/details/taskInstance/RenderedK8s.tsx
@@ -0,0 +1,55 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { useRef } from "react";
+import { Code } from "@chakra-ui/react";
+import YAML from "json-to-pretty-yaml";
+
+import { getMetaValue, useOffsetTop } from "src/utils";
+
+import useSelection from "src/dag/useSelection";
+import { useRenderedK8s } from "src/api";
+
+const isK8sExecutor = getMetaValue("k8s_or_k8scelery_executor") === "True";
+
+const RenderedK8s = () => {
+ const {
+ selected: { runId, taskId, mapIndex },
+ } = useSelection();
+
+ const { data: renderedK8s } = useRenderedK8s(runId, taskId, mapIndex);
+
+ const k8sRef = useRef<HTMLPreElement>(null);
+ const offsetTop = useOffsetTop(k8sRef);
+
+ if (!isK8sExecutor || !runId || !taskId) return null;
+
+ return (
+ <Code
+ mt={3}
+ ref={k8sRef}
+ maxHeight={`calc(100% - ${offsetTop}px)`}
+ overflowY="auto"
+ >
+ <pre>{YAML.stringify(renderedK8s)}</pre>
+ </Code>
+ );
+};
+
+export default RenderedK8s;
diff --git a/airflow/www/templates/airflow/dag.html
b/airflow/www/templates/airflow/dag.html
index 29f535b4b6..c91d4b4454 100644
--- a/airflow/www/templates/airflow/dag.html
+++ b/airflow/www/templates/airflow/dag.html
@@ -65,7 +65,7 @@
<meta name="graph_url" content="{{ url_for('Airflow.graph',
dag_id=dag.dag_id, root=root) }}">
<meta name="task_url" content="{{ url_for('Airflow.task', dag_id=dag.dag_id)
}}">
<meta name="log_url" content="{{ url_for('Airflow.log', dag_id=dag.dag_id)
}}">
- <meta name="rendered_k8s_url" content="{{ url_for('Airflow.rendered_k8s',
dag_id=dag.dag_id) }}">
+ <meta name="rendered_k8s_data_url" content="{{
url_for('Airflow.rendered_k8s_data', dag_id=dag.dag_id) }}">
<meta name="task_instances_list_url" content="{{
url_for('TaskInstanceModelView.list') }}">
<meta name="tag_index_url" content="{{ url_for('Airflow.index',
tags='_TAG_NAME_') }}">
<meta name="mapped_instances_api" content="{{
url_for('/api/v1.airflow_api_connexion_endpoints_task_instance_endpoint_get_mapped_task_instances',
dag_id=dag.dag_id, dag_run_id='_DAG_RUN_ID_', task_id='_TASK_ID_') }}">
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 306926c163..b2f8c2788b 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -1493,7 +1493,7 @@ class Airflow(AirflowBaseView):
form = DateTimeForm(data={"execution_date": dttm})
root = request.args.get("root", "")
map_index = request.args.get("map_index", -1, type=int)
- logger.info("Retrieving rendered templates.")
+ logger.info("Retrieving rendered k8s.")
dag: DAG = get_airflow_app().dag_bag.get_dag(dag_id)
task = dag.get_task(task_id)
@@ -1538,6 +1538,48 @@ class Airflow(AirflowBaseView):
title=title,
)
+ @expose("/object/rendered-k8s")
+ @auth.has_access_dag("GET", DagAccessEntity.TASK_INSTANCE)
+ @provide_session
+ def rendered_k8s_data(self, *, session: Session = NEW_SESSION):
+ """Get rendered k8s yaml."""
+ if not settings.IS_K8S_OR_K8SCELERY_EXECUTOR:
+ return {"error": "Not a k8s or k8s_celery executor"}, 404
+ # This part is only used for k8s executor so providers.cncf.kubernetes
must be installed
+ # with the get_rendered_k8s_spec method
+ from airflow.providers.cncf.kubernetes.template_rendering import
get_rendered_k8s_spec
+
+ dag_id = request.args.get("dag_id")
+ task_id = request.args.get("task_id")
+ if task_id is None:
+ return {"error": "Task id not passed in the request"}, 404
+ run_id = request.args.get("run_id")
+ map_index = request.args.get("map_index", -1, type=int)
+ logger.info("Retrieving rendered k8s data.")
+
+ dag: DAG = get_airflow_app().dag_bag.get_dag(dag_id)
+ task = dag.get_task(task_id)
+ dag_run = dag.get_dagrun(run_id=run_id, session=session)
+ ti = dag_run.get_task_instance(task_id=task.task_id,
map_index=map_index, session=session)
+
+ if not ti:
+ return {"error": f"can't find task instance {task.task_id}"}, 404
+ pod_spec = None
+ if not isinstance(ti, TaskInstance):
+ return {"error": f"{task.task_id} is not a task instance"}, 500
+ try:
+ pod_spec = get_rendered_k8s_spec(ti, session=session)
+ except AirflowException as e:
+ if not e.__cause__:
+ return {"error": f"Error rendering Kubernetes POD Spec: {e}"},
500
+ else:
+ tmp = Markup("Error rendering Kubernetes POD Spec:
{0}<br><br>Original error: {0.__cause__}")
+ return {"error": tmp.format(e)}, 500
+ except Exception as e:
+ return {"error": f"Error rendering Kubernetes Pod Spec: {e}"}, 500
+
+ return pod_spec
+
@expose("/get_logs_with_metadata")
@auth.has_access_dag("GET", DagAccessEntity.TASK_INSTANCE)
@auth.has_access_dag("GET", DagAccessEntity.TASK_LOGS)
diff --git a/airflow/www/yarn.lock b/airflow/www/yarn.lock
index b4ec5af7a2..73056df0e6 100644
--- a/airflow/www/yarn.lock
+++ b/airflow/www/yarn.lock
@@ -3360,6 +3360,11 @@
resolved
"https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.7.tgz#98a993516c859eb0d5c4c8f098317a9ea68db9ad";
integrity
sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==
+"@types/json-to-pretty-yaml@^1.2.1":
+ version "1.2.1"
+ resolved
"https://registry.yarnpkg.com/@types/json-to-pretty-yaml/-/json-to-pretty-yaml-1.2.1.tgz#bf193455477295d83c78f73c08d956f74321193e";
+ integrity
sha512-+uOlBkCPkny6CE2a5IAR0Q21/ZE+90MsK7EfDblDdutcey+rbMDrp3i93M6MTwbMHFB75aIFR5fVXVcnLCkAiw==
+
"@types/json5@^0.0.29":
version "0.0.29"
resolved
"https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee";
@@ -8006,6 +8011,14 @@ json-stringify-safe@^5.0.1:
resolved
"https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb";
integrity
sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==
+json-to-pretty-yaml@^1.2.2:
+ version "1.2.2"
+ resolved
"https://registry.yarnpkg.com/json-to-pretty-yaml/-/json-to-pretty-yaml-1.2.2.tgz#f4cd0bd0a5e8fe1df25aaf5ba118b099fd992d5b";
+ integrity
sha512-rvm6hunfCcqegwYaG5T4yKJWxc9FXFgBVrcTZ4XfSVRwa5HA/Xs+vB/Eo9treYYHCeNM0nrSUr82V/M31Urc7A==
+ dependencies:
+ remedial "^1.0.7"
+ remove-trailing-spaces "^1.0.6"
+
json5@^1.0.2:
version "1.0.2"
resolved
"https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593";
@@ -9875,10 +9888,10 @@ react-focus-lock@^2.9.1:
use-callback-ref "^1.3.0"
use-sidecar "^1.1.2"
-react-icons@^4.9.0:
- version "4.9.0"
- resolved
"https://registry.yarnpkg.com/react-icons/-/react-icons-4.9.0.tgz#ba44f436a053393adb1bdcafbc5c158b7b70d2a3";
- integrity
sha512-ijUnFr//ycebOqujtqtV9PFS7JjhWg0QU6ykURVHuL4cbofvRCf3f6GMn9+fBktEFQOIVZnuAYLZdiyadRQRFg==
+react-icons@^5.1.0:
+ version "5.1.0"
+ resolved
"https://registry.yarnpkg.com/react-icons/-/react-icons-5.1.0.tgz#9e7533cc256571a610c2a1ec8a7a143fb1222943";
+ integrity
sha512-D3zug1270S4hbSlIRJ0CUS97QE1yNNKDjzQe3HqY0aefp2CBn9VgzgES27sRR2gOvFK+0CNx/BW0ggOESp6fqQ==
react-is@^16.13.1, react-is@^16.7.0, react-is@^16.8.1:
version "16.13.1"
@@ -10248,11 +10261,21 @@ remark-rehype@^10.0.0:
mdast-util-to-hast "^12.1.0"
unified "^10.0.0"
+remedial@^1.0.7:
+ version "1.0.8"
+ resolved
"https://registry.yarnpkg.com/remedial/-/remedial-1.0.8.tgz#a5e4fd52a0e4956adbaf62da63a5a46a78c578a0";
+ integrity
sha512-/62tYiOe6DzS5BqVsNpH/nkGlX45C/Sp6V+NtiN6JQNS1Viay7cWkazmRkrQrdFj2eshDe96SIQNIoMxqhzBOg==
+
[email protected]:
version "0.4.2"
resolved
"https://registry.yarnpkg.com/remove-accents/-/remove-accents-0.4.2.tgz#0a43d3aaae1e80db919e07ae254b285d9e1c7bb5";
integrity sha1-CkPTqq4egNuRngeuJUsoXZ4ce7U=
+remove-trailing-spaces@^1.0.6:
+ version "1.0.8"
+ resolved
"https://registry.yarnpkg.com/remove-trailing-spaces/-/remove-trailing-spaces-1.0.8.tgz#4354d22f3236374702f58ee373168f6d6887ada7";
+ integrity
sha512-O3vsMYfWighyFbTd8hk8VaSj9UAGENxAtX+//ugIst2RMk5e03h6RoIS+0ylsFxY1gvmPuAY/PO4It+gPEeySA==
+
require-directory@^2.1.1:
version "2.1.1"
resolved
"https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42";
