This is an automated email from the ASF dual-hosted git repository.
phanikumv pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 32c88a2906 Add exclude/include events filters to audit log (#38506)
32c88a2906 is described below
commit 32c88a29060ee9d1a083d5ea1d970a99f3d5ddeb
Author: Brent Bovenzi <[email protected]>
AuthorDate: Tue Apr 2 04:48:59 2024 -0400
Add exclude/include events filters to audit log (#38506)
Co-authored-by: Wei Lee <[email protected]>
---
airflow/www/extensions/init_jinja_globals.py | 2 +
airflow/www/static/js/api/useEventLogs.tsx | 12 ++++
airflow/www/static/js/dag/details/AuditLog.tsx | 83 ++++++++++++++++++++++++--
airflow/www/static/js/datasets/SearchBar.tsx | 77 ++++++++++++------------
airflow/www/templates/airflow/dag.html | 2 +
airflow/www/views.py | 5 ++
tests/www/views/test_views.py | 2 +
7 files changed, 142 insertions(+), 41 deletions(-)
diff --git a/airflow/www/extensions/init_jinja_globals.py
b/airflow/www/extensions/init_jinja_globals.py
index e13a46be7b..a64212ef7b 100644
--- a/airflow/www/extensions/init_jinja_globals.py
+++ b/airflow/www/extensions/init_jinja_globals.py
@@ -76,6 +76,8 @@ def init_jinja_globals(app):
"k8s_or_k8scelery_executor": IS_K8S_OR_K8SCELERY_EXECUTOR,
"rest_api_enabled": False,
"config_test_connection": conf.get("core", "test_connection",
fallback="Disabled"),
+ "included_events_raw": conf.get("webserver",
"audit_view_included_events", fallback=""),
+ "excluded_events_raw": conf.get("webserver",
"audit_view_excluded_events", fallback=""),
}
# Extra global specific to auth manager
diff --git a/airflow/www/static/js/api/useEventLogs.tsx
b/airflow/www/static/js/api/useEventLogs.tsx
index 3a04eac393..4b5cc92ce5 100644
--- a/airflow/www/static/js/api/useEventLogs.tsx
+++ b/airflow/www/static/js/api/useEventLogs.tsx
@@ -34,6 +34,8 @@ export default function useEventLogs({
after,
before,
owner,
+ includedEvents,
+ excludedEvents,
}: API.GetEventLogsVariables) {
const { isRefreshOn } = useAutoRefresh();
return useQuery(
@@ -48,10 +50,18 @@ export default function useEventLogs({
after,
before,
owner,
+ excludedEvents,
+ includedEvents,
],
() => {
const eventsLogUrl = getMetaValue("event_logs_api");
const orderParam = orderBy ? { order_by: orderBy } : {};
+ const excludedParam = excludedEvents
+ ? { excluded_events: excludedEvents }
+ : {};
+ const includedParam = includedEvents
+ ? { included_events: includedEvents }
+ : {};
return axios.get<AxiosResponse, API.EventLogCollection>(eventsLogUrl, {
params: {
offset,
@@ -60,6 +70,8 @@ export default function useEventLogs({
...{ task_id: taskId },
...{ run_id: runId },
...orderParam,
+ ...excludedParam,
+ ...includedParam,
after,
before,
},
diff --git a/airflow/www/static/js/dag/details/AuditLog.tsx
b/airflow/www/static/js/dag/details/AuditLog.tsx
index 11f5447a98..e30fbbe68e 100644
--- a/airflow/www/static/js/dag/details/AuditLog.tsx
+++ b/airflow/www/static/js/dag/details/AuditLog.tsx
@@ -19,7 +19,7 @@
/* global moment */
-import React, { useMemo, useRef } from "react";
+import React, { useMemo, useRef, useState } from "react";
import {
Box,
Flex,
@@ -27,12 +27,20 @@ import {
FormHelperText,
FormLabel,
Input,
- HStack,
+ SimpleGrid,
Button,
} from "@chakra-ui/react";
import { createColumnHelper } from "@tanstack/react-table";
import { snakeCase } from "lodash";
+import {
+ OptionBase,
+ useChakraSelectProps,
+ CreatableSelect,
+ GroupBase,
+ ChakraStylesConfig,
+} from "chakra-react-select";
+
import { useEventLogs } from "src/api";
import { getMetaValue, useOffsetTop } from "src/utils";
import type { DagRun } from "src/types";
@@ -43,11 +51,19 @@ import { useTableURLState } from
"src/components/NewTable/useTableUrlState";
import { CodeCell, TimeCell } from "src/components/NewTable/NewCells";
import { MdRefresh } from "react-icons/md";
+const configExcludedEvents = getMetaValue("excluded_audit_log_events");
+const configIncludedEvents = getMetaValue("included_audit_log_events");
+
interface Props {
taskId?: string;
run?: DagRun;
}
+interface Option extends OptionBase {
+ label: string;
+ value: string;
+}
+
const dagId = getMetaValue("dag_id") || undefined;
const columnHelper = createColumnHelper<EventLog>();
@@ -58,6 +74,12 @@ const AuditLog = ({ taskId, run }: Props) => {
const { tableURLState, setTableURLState } = useTableURLState({
sorting: [{ id: "when", desc: true }],
});
+ const [includedEvents, setIncludedEvents] = useState(
+ configIncludedEvents.length ? configIncludedEvents.split(",") : []
+ );
+ const [excludedEvents, setExcludedEvents] = useState(
+ configExcludedEvents.length ? configExcludedEvents.split(",") : []
+ );
const sort = tableURLState.sorting[0];
const orderBy = sort ? `${sort.desc ? "-" : ""}${snakeCase(sort.id)}` : "";
@@ -72,6 +94,8 @@ const AuditLog = ({ taskId, run }: Props) => {
limit: tableURLState.pagination.pageSize,
offset:
tableURLState.pagination.pageIndex * tableURLState.pagination.pageSize,
+ includedEvents: includedEvents ? includedEvents.join(",") : undefined,
+ excludedEvents: excludedEvents ? excludedEvents.join(",") : undefined,
});
const columns = useMemo(() => {
@@ -116,6 +140,49 @@ const AuditLog = ({ taskId, run }: Props) => {
const memoData = useMemo(() => data?.eventLogs, [data?.eventLogs]);
+ const chakraStyles: ChakraStylesConfig<Option, true, GroupBase<Option>> = {
+ dropdownIndicator: (provided) => ({
+ ...provided,
+ display: "none",
+ }),
+ indicatorSeparator: (provided) => ({
+ ...provided,
+ display: "none",
+ }),
+ menuList: (provided) => ({
+ ...provided,
+ py: 0,
+ }),
+ };
+
+ const excludedEventsSelectProps = useChakraSelectProps<Option, true>({
+ isMulti: true,
+ tagVariant: "solid",
+ value: excludedEvents.map((e) => ({
+ label: e,
+ value: e,
+ })),
+ onChange: (options) => {
+ setExcludedEvents((options || []).map(({ value }) => value));
+ },
+ placeholder: "Type to filter an event",
+ chakraStyles,
+ });
+
+ const includedEventsSelectProps = useChakraSelectProps<Option, true>({
+ isMulti: true,
+ tagVariant: "solid",
+ value: includedEvents.map((e) => ({
+ label: e,
+ value: e,
+ })),
+ onChange: (options) => {
+ setIncludedEvents((options || []).map(({ value }) => value));
+ },
+ placeholder: "Type to filter an event",
+ chakraStyles,
+ });
+
return (
<Box
height="100%"
@@ -137,7 +204,7 @@ const AuditLog = ({ taskId, run }: Props) => {
View full cluster Audit Log
</LinkButton>
</Flex>
- <HStack spacing={2} alignItems="flex-start">
+ <SimpleGrid columns={4} columnGap={2}>
<FormControl>
<FormLabel>Show Logs After</FormLabel>
<Input
@@ -178,7 +245,15 @@ const AuditLog = ({ taskId, run }: Props) => {
<Input placeholder={taskId} isDisabled />
<FormHelperText />
</FormControl>
- </HStack>
+ <FormControl>
+ <FormLabel>Events to exclude</FormLabel>
+ <CreatableSelect {...excludedEventsSelectProps} />
+ </FormControl>
+ <FormControl>
+ <FormLabel>Events to include</FormLabel>
+ <CreatableSelect {...includedEventsSelectProps} />
+ </FormControl>
+ </SimpleGrid>
<NewTable
key={`${taskId}-${run?.runId}`}
data={memoData || []}
diff --git a/airflow/www/static/js/datasets/SearchBar.tsx
b/airflow/www/static/js/datasets/SearchBar.tsx
index 094bb20985..4702035bf2 100644
--- a/airflow/www/static/js/datasets/SearchBar.tsx
+++ b/airflow/www/static/js/datasets/SearchBar.tsx
@@ -18,10 +18,9 @@
*/
import React from "react";
-import { Size, useChakraSelectProps } from "chakra-react-select";
+import { Select, SingleValue, useChakraSelectProps } from
"chakra-react-select";
import type { DatasetDependencies } from "src/api/useDatasetDependencies";
-import MultiSelect from "src/components/MultiSelect";
interface Props {
datasetDependencies?: DatasetDependencies;
@@ -50,17 +49,47 @@ const SearchBar = ({
datasetOptions.push({ value: node.id, label: node.value.label });
});
- const inputStyles: { backgroundColor: string; size: Size } = {
- backgroundColor: "white",
- size: "lg",
+ const onSelect = (option: SingleValue<Option>) => {
+ let type = "";
+ if (option) {
+ if (option.value.startsWith("dataset:")) type = "dataset";
+ else if (option.value.startsWith("dag:")) type = "dag";
+ if (type) onSelectNode(option.label, type);
+ }
};
- const selectStyles = useChakraSelectProps({
- ...inputStyles,
- tagVariant: "solid",
- // hideSelectedOptions: false,
- // isClearable: false,
+
+ let option;
+ if (selectedUri) {
+ option = { label: selectedUri, value: `dataset:${selectedUri}` };
+ } else if (selectedDagId) {
+ option = { label: selectedDagId, value: `dag:${selectedDagId}` };
+ }
+
+ const searchProps = useChakraSelectProps<Option, false>({
selectedOptionStyle: "check",
+ isDisabled: !datasetDependencies,
+ value: option,
+ onChange: onSelect,
+ options: [
+ { label: "DAGs", options: dagOptions },
+ { label: "Datasets", options: datasetOptions },
+ ],
+ placeholder: "Search by DAG ID or Dataset URI",
chakraStyles: {
+ dropdownIndicator: (provided) => ({
+ ...provided,
+ bg: "transparent",
+ px: 2,
+ cursor: "inherit",
+ }),
+ indicatorSeparator: (provided) => ({
+ ...provided,
+ display: "none",
+ }),
+ menuList: (provided) => ({
+ ...provided,
+ py: 0,
+ }),
container: (p) => ({
...p,
width: "100%",
@@ -93,33 +122,7 @@ const SearchBar = ({
},
});
- const onSelect = ({ label, value }: Option) => {
- let type = "";
- if (value.startsWith("dataset:")) type = "dataset";
- else if (value.startsWith("dag:")) type = "dag";
- if (type) onSelectNode(label, type);
- };
-
- let option;
- if (selectedUri) {
- option = { label: selectedUri, value: `dataset:${selectedUri}` };
- } else if (selectedDagId) {
- option = { label: selectedDagId, value: `dag:${selectedDagId}` };
- }
-
- return (
- <MultiSelect
- {...selectStyles}
- isDisabled={!datasetDependencies}
- value={option}
- onChange={(e) => onSelect(e as Option)}
- options={[
- { label: "DAGs", options: dagOptions },
- { label: "Datasets", options: datasetOptions },
- ]}
- placeholder="Search by DAG ID or Dataset URI"
- />
- );
+ return <Select {...searchProps} />;
};
export default SearchBar;
diff --git a/airflow/www/templates/airflow/dag.html
b/airflow/www/templates/airflow/dag.html
index 9cb4d430a6..56bfc27070 100644
--- a/airflow/www/templates/airflow/dag.html
+++ b/airflow/www/templates/airflow/dag.html
@@ -89,6 +89,8 @@
<meta name="is_paused" content="{{ dag_is_paused }}">
<meta name="csrf_token" content="{{ csrf_token() }}">
<meta name="k8s_or_k8scelery_executor" content="{{ k8s_or_k8scelery_executor
}}">
+ <meta name="excluded_audit_log_events" content="{{ excluded_events_raw }}">
+ <meta name="included_audit_log_events" content="{{ included_events_raw }}">
{% if dag_model is defined and dag_model.next_dagrun_create_after is defined
and dag_model.next_dagrun_create_after is not none %}
<meta name="next_dagrun_create_after" content="{{
dag_model.next_dagrun_create_after }}">
<meta name="next_dagrun_data_interval_start" content="{{
dag_model.next_dagrun_data_interval_start }}">
diff --git a/airflow/www/views.py b/airflow/www/views.py
index a9b3d65ca1..8a5ea38270 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2796,6 +2796,9 @@ class Airflow(AirflowBaseView):
wwwutils.check_import_errors(dag.fileloc, session)
wwwutils.check_dag_warnings(dag.dag_id, session)
+ included_events_raw = conf.get("webserver",
"audit_view_included_events", fallback="")
+ excluded_events_raw = conf.get("webserver",
"audit_view_excluded_events", fallback="")
+
root = request.args.get("root")
if root:
dag = dag.partial_subset(task_ids_or_regex=root,
include_downstream=False, include_upstream=True)
@@ -2840,6 +2843,8 @@ class Airflow(AirflowBaseView):
"numRuns": num_runs_options,
}
),
+ included_events_raw=included_events_raw,
+ excluded_events_raw=excluded_events_raw,
)
@expose("/calendar")
diff --git a/tests/www/views/test_views.py b/tests/www/views/test_views.py
index a588cb1864..27f096403f 100644
--- a/tests/www/views/test_views.py
+++ b/tests/www/views/test_views.py
@@ -94,6 +94,8 @@ def test_redoc_should_render_template(capture_templates,
admin_client):
"openapi_spec_url": "/api/v1/openapi.yaml",
"rest_api_enabled": True,
"get_docs_url": get_docs_url,
+ "excluded_events_raw": "",
+ "included_events_raw": "",
}
