This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 4bd4f5790c [INLONG-8693][Dashboard] Data synchronization supports
Audit (#8704)
4bd4f5790c is described below
commit 4bd4f5790c5925b5fb2e7a3988d46ae6f084a1b5
Author: Lizhen <[email protected]>
AuthorDate: Sun Aug 13 16:34:12 2023 +0800
[INLONG-8693][Dashboard] Data synchronization supports Audit (#8704)
---
inlong-dashboard/src/ui/locales/cn.json | 3 +
inlong-dashboard/src/ui/locales/en.json | 3 +
.../pages/SynchronizeDetail/SyncAudit/config.tsx | 215 +++++++++++++++++++++
.../ui/pages/SynchronizeDetail/SyncAudit/index.tsx | 148 ++++++++++++++
.../src/ui/pages/SynchronizeDetail/index.tsx | 34 ++--
5 files changed, 390 insertions(+), 13 deletions(-)
diff --git a/inlong-dashboard/src/ui/locales/cn.json
b/inlong-dashboard/src/ui/locales/cn.json
index b0b6dadc89..a04bafc10c 100644
--- a/inlong-dashboard/src/ui/locales/cn.json
+++ b/inlong-dashboard/src/ui/locales/cn.json
@@ -781,6 +781,7 @@
"pages.Nodes.TestConnection": "测试连接",
"pages.SynchronizeDetail.Info": "基本信息",
"pages.SynchronizeDetail.SourceAndSink": "配置数据流",
+ "pages.SynchronizeDetail.Audit": "审计",
"pages.SynchronizeDashboard.Create": "新建数据同步",
"pages.SynchronizeDashboard.config.ExecuteLog": "日志",
"pages.SynchronizeDashboard.config.Restart": "重启",
@@ -802,6 +803,8 @@
"pages.SynchronizeDetail.Sync.Transform": "设置 Transform",
"pages.SynchronizeDetail.Transform": "数据转换",
"pages.SynchronizeDetail.Transform.Create": "新建数据转换",
+ "pages.SynchronizeDetail.Audit.Read": "读取成功",
+ "pages.SynchronizeDetail.Audit.Receive": "接收成功",
"components.FieldList.Source": "源字段",
"components.FieldList.Sink": "目标字段",
"components.FieldList.CreateSource": "新建源字段",
diff --git a/inlong-dashboard/src/ui/locales/en.json
b/inlong-dashboard/src/ui/locales/en.json
index 6f7b2445ed..5a2bc56b79 100644
--- a/inlong-dashboard/src/ui/locales/en.json
+++ b/inlong-dashboard/src/ui/locales/en.json
@@ -781,6 +781,7 @@
"pages.Nodes.TestConnection": "Test Connection",
"pages.SynchronizeDetail.Info": "Info",
"pages.SynchronizeDetail.SourceAndSink": "DataStream",
+ "pages.SynchronizeDetail.Audit": "Audit",
"pages.SynchronizeDashboard.Create": "Create",
"pages.SynchronizeDashboard.config.Total": "Total",
"pages.SynchronizeDashboard.config.WaitAssignCount": "WaitAssignCount",
@@ -802,6 +803,8 @@
"pages.SynchronizeDetail.Sync.Transform": "Transform",
"pages.SynchronizeDetail.Transform": "Transform",
"pages.SynchronizeDetail.Transform.Create": "Create",
+ "pages.SynchronizeDetail.Audit.Read": "Read",
+ "pages.SynchronizeDetail.Audit.Receive": "Receive",
"components.FieldList.Source": "SourceField",
"components.FieldList.Sink": "SinkField",
"components.FieldList.CreateSource": "Create SourceField",
diff --git
a/inlong-dashboard/src/ui/pages/SynchronizeDetail/SyncAudit/config.tsx
b/inlong-dashboard/src/ui/pages/SynchronizeDetail/SyncAudit/config.tsx
new file mode 100644
index 0000000000..2326dcf430
--- /dev/null
+++ b/inlong-dashboard/src/ui/pages/SynchronizeDetail/SyncAudit/config.tsx
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React from 'react';
+import { Button } from 'antd';
+import dayjs from 'dayjs';
+import i18n from '@/i18n';
+import { sinks } from '@/plugins/sinks';
+import { sources } from '@/plugins/sources';
+
+export const timeStaticsDimList = [
+ {
+ label: i18n.t('pages.GroupDetail.Audit.Min'),
+ value: 'MINUTE',
+ },
+ {
+ label: i18n.t('pages.GroupDetail.Audit.Hour'),
+ value: 'HOUR',
+ },
+ {
+ label: i18n.t('pages.GroupDetail.Audit.Day'),
+ value: 'DAY',
+ },
+];
+
+const auditList = [
+ {
+ label: `Source ${i18n.t('pages.SynchronizeDetail.Audit.Read')}`,
+ value: 1,
+ },
+ {
+ label: `Sink ${i18n.t('pages.SynchronizeDetail.Audit.Receive')}`,
+ value: 2,
+ },
+];
+
+function getAuditLabel(auditId: number, nodeType?: string) {
+ const id = +auditId;
+ const item = auditList[id % 2 ? 0 : 1];
+ const label = item?.label || id;
+ const sourceLabel = sources.find(c => c.value === nodeType)?.label;
+ const sinkLabel = sinks.find(c => c.value === nodeType)?.label;
+ return item?.value === 1 ? `${label}(${sourceLabel})` :
`${label}(${sinkLabel})`;
+}
+
+export const toChartData = (source, sourceDataMap) => {
+ const xAxisData = Object.keys(sourceDataMap);
+ return {
+ legend: {
+ data: source.map(item => getAuditLabel(item.auditId, item.nodeType)),
+ },
+ tooltip: {
+ trigger: 'axis',
+ },
+ xAxis: {
+ type: 'category',
+ data: xAxisData,
+ },
+ yAxis: {
+ type: 'value',
+ },
+ series: source.map(item => ({
+ name: getAuditLabel(item.auditId, item.nodeType),
+ type: 'line',
+ data: xAxisData.map(logTs => sourceDataMap[logTs]?.[item.auditId] || 0),
+ })),
+ };
+};
+
+export const toTableData = (source, sourceDataMap) => {
+ return Object.keys(sourceDataMap)
+ .reverse()
+ .map(logTs => ({
+ ...sourceDataMap[logTs],
+ logTs,
+ }));
+};
+
+export const getFormContent = (inlongGroupId, initialValues, onSearch,
onDataStreamSuccess) => [
+ {
+ type: 'select',
+ label: i18n.t('pages.GroupDetail.Audit.DataStream'),
+ name: 'inlongStreamId',
+ props: {
+ dropdownMatchSelectWidth: false,
+ options: {
+ requestAuto: true,
+ requestService: {
+ url: '/stream/list',
+ method: 'POST',
+ data: {
+ pageNum: 1,
+ pageSize: 1000,
+ inlongGroupId,
+ },
+ },
+ requestParams: {
+ formatResult: result =>
+ result?.list.map(item => ({
+ label: item.inlongStreamId,
+ value: item.inlongStreamId,
+ })) || [],
+ onSuccess: onDataStreamSuccess,
+ },
+ },
+ },
+ rules: [{ required: true }],
+ },
+ {
+ type: 'select',
+ label: i18n.t('pages.GroupDetail.Audit.Sink'),
+ name: 'sinkId',
+ props: values => ({
+ dropdownMatchSelectWidth: false,
+ options: {
+ requestService: {
+ url: '/sink/list',
+ method: 'POST',
+ data: {
+ pageNum: 1,
+ pageSize: 1000,
+ inlongGroupId,
+ inlongStreamId: values.inlongStreamId,
+ },
+ },
+ requestParams: {
+ formatResult: result =>
+ result?.list.map(item => ({
+ label: item.sinkName + ` ( ${sinks.find(c => c.value ===
item.sinkType)?.label} )`,
+ value: item.id,
+ })) || [],
+ },
+ },
+ }),
+ },
+ {
+ type: 'datepicker',
+ label: i18n.t('pages.GroupDetail.Audit.StartDate'),
+ name: 'startDate',
+ initialValue: dayjs(initialValues.startDate),
+ props: {
+ allowClear: false,
+ format: 'YYYY-MM-DD',
+ },
+ },
+ {
+ type: 'datepicker',
+ label: i18n.t('pages.GroupDetail.Audit.EndDate'),
+ name: 'endDate',
+ initialValues: dayjs(initialValues.endDate),
+ props: {
+ allowClear: false,
+ format: 'YYYY-MM-DD',
+ disabledDate: current => {
+ const start = dayjs(initialValues.startDate);
+ const dim = initialValues.timeStaticsDim;
+ if (dim === 'HOUR' || dim === 'DAY') {
+ const tooLate = current && current <= start.endOf('day');
+ const tooEarly = start && current > start.add(7, 'd').endOf('day');
+ return tooLate || tooEarly;
+ }
+ const tooLate = current && current >= start.endOf('day');
+ const tooEarly = start && current < start.add(-1, 'd').endOf('day');
+ return tooLate || tooEarly;
+ },
+ },
+ },
+ {
+ type: 'select',
+ label: i18n.t('pages.GroupDetail.Audit.TimeStaticsDim'),
+ name: 'timeStaticsDim',
+ initialValue: initialValues.timeStaticsDim,
+ props: {
+ dropdownMatchSelectWidth: false,
+ options: timeStaticsDimList,
+ },
+ },
+ {
+ type: (
+ <Button type="primary" onClick={onSearch}>
+ {i18n.t('pages.GroupDetail.Audit.Search')}
+ </Button>
+ ),
+ },
+];
+
+export const getTableColumns = source => {
+ const data = source.map(item => ({
+ title: getAuditLabel(item.auditId, item.nodeType),
+ dataIndex: item.auditId,
+ render: text => text || 0,
+ }));
+ return [
+ {
+ title: i18n.t('pages.GroupDetail.Audit.Time'),
+ dataIndex: 'logTs',
+ },
+ ].concat(data);
+};
diff --git
a/inlong-dashboard/src/ui/pages/SynchronizeDetail/SyncAudit/index.tsx
b/inlong-dashboard/src/ui/pages/SynchronizeDetail/SyncAudit/index.tsx
new file mode 100644
index 0000000000..568c89d530
--- /dev/null
+++ b/inlong-dashboard/src/ui/pages/SynchronizeDetail/SyncAudit/index.tsx
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import React, { useMemo, useState } from 'react';
+import FormGenerator, { useForm } from '@/ui/components/FormGenerator';
+import HighTable from '@/ui/components/HighTable';
+import { useRequest } from '@/ui/hooks';
+import { timestampFormat } from '@/core/utils';
+import Charts from '@/ui/components/Charts';
+import { CommonInterface } from '../common';
+import {
+ getFormContent,
+ toChartData,
+ toTableData,
+ getTableColumns,
+ timeStaticsDimList,
+} from './config';
+
+type Props = CommonInterface;
+
+const Comp: React.FC<Props> = ({ inlongGroupId }) => {
+ const [form] = useForm();
+
+ const [query, setQuery] = useState({
+ inlongStreamId: '',
+ startDate: +new Date(),
+ endDate: +new Date(),
+ timeStaticsDim: timeStaticsDimList[0].value,
+ });
+
+ const { data: sourceData = [], run } = useRequest(
+ {
+ url: '/audit/list',
+ method: 'POST',
+ data: {
+ ...query,
+ startDate: timestampFormat(query.startDate, 'yyyy-MM-dd'),
+ endDate: timestampFormat(query.endDate, 'yyyy-MM-dd'),
+ inlongGroupId,
+ },
+ },
+ {
+ ready: Boolean(query.inlongStreamId),
+ formatResult: result => result.sort((a, b) => (a.auditId - b.auditId > 0
? 1 : -1)),
+ },
+ );
+
+ const sourceDataMap = useMemo(() => {
+ const data = sourceData.filter(item => item.auditId >= 9);
+ const flatArr = data
+ .reduce(
+ (acc, cur) =>
+ acc.concat(
+ cur.auditSet.map(item => ({
+ ...item,
+ auditId: cur.auditId,
+ })),
+ ),
+ [],
+ )
+ .sort((a, b) => {
+ const aT = +new Date(query.timeStaticsDim === 'HOUR' ? `${a.logTs}:00`
: a.logTs);
+ const bT = +new Date(query.timeStaticsDim === 'HOUR' ? `${b.logTs}:00`
: b.logTs);
+ return aT - bT;
+ });
+ const output = flatArr.reduce((acc, cur) => {
+ if (!acc[cur.logTs]) {
+ acc[cur.logTs] = {};
+ }
+ acc[cur.logTs] = {
+ ...acc[cur.logTs],
+ [cur.auditId]: cur.count,
+ };
+ return acc;
+ }, {});
+ return output;
+ }, [sourceData, query.timeStaticsDim]);
+
+ const onSearch = async () => {
+ await form.validateFields();
+ run();
+ };
+
+ const onDataStreamSuccess = data => {
+ const defaultDataStream = data[0]?.value;
+ if (defaultDataStream) {
+ form.setFieldsValue({ inlongStreamId: defaultDataStream });
+ setQuery(prev => ({ ...prev, inlongStreamId: defaultDataStream }));
+ run();
+ }
+ };
+
+ return (
+ <>
+ <div style={{ marginBottom: 40 }}>
+ <FormGenerator
+ form={form}
+ layout="inline"
+ content={getFormContent(inlongGroupId, query, onSearch,
onDataStreamSuccess)}
+ style={{ marginBottom: 30 }}
+ onFilter={allValues =>
+ setQuery({
+ ...allValues,
+ startDate: +allValues.startDate.$d,
+ endDate: +allValues.startDate.$d,
+ })
+ }
+ />
+ <Charts
+ height={400}
+ option={toChartData(
+ sourceData.filter(item => item.auditId >= 9),
+ sourceDataMap,
+ )}
+ />
+ </div>
+
+ <HighTable
+ table={{
+ columns: getTableColumns(sourceData.filter(item => item.auditId >=
9)),
+ dataSource: toTableData(
+ sourceData.filter(item => item.auditId >= 9),
+ sourceDataMap,
+ ),
+ rowKey: 'logTs',
+ }}
+ />
+ </>
+ );
+};
+
+export default Comp;
diff --git a/inlong-dashboard/src/ui/pages/SynchronizeDetail/index.tsx
b/inlong-dashboard/src/ui/pages/SynchronizeDetail/index.tsx
index cde535ca17..31f400c32c 100644
--- a/inlong-dashboard/src/ui/pages/SynchronizeDetail/index.tsx
+++ b/inlong-dashboard/src/ui/pages/SynchronizeDetail/index.tsx
@@ -26,6 +26,7 @@ import { useTranslation } from 'react-i18next';
import request from '@/core/utils/request';
import Info from './Info';
import DataStream from './SyncT';
+import SyncAudit from './SyncAudit';
import i18n from '@/i18n';
const Comp: React.FC = () => {
@@ -60,19 +61,26 @@ const Comp: React.FC = () => {
const isReadonly = useMemo(() => [0, 101, 102].includes(data?.status),
[data]);
const list = useMemo(
- () => [
- {
- label: t('pages.SynchronizeDetail.Info'),
- value: 'syncInfo',
- content: Info,
- },
- {
- label: t('pages.SynchronizeDetail.SourceAndSink'),
- value: 'syncSource',
- content: DataStream,
- },
- ],
- [],
+ () =>
+ [
+ {
+ label: t('pages.SynchronizeDetail.Info'),
+ value: 'syncInfo',
+ content: Info,
+ },
+ {
+ label: t('pages.SynchronizeDetail.SourceAndSink'),
+ value: 'syncSource',
+ content: DataStream,
+ },
+ {
+ label: t('pages.SynchronizeDetail.Audit'),
+ value: 'syncAudit',
+ content: SyncAudit,
+ hidden: isReadonly || isCreate,
+ },
+ ].filter(item => !item.hidden),
+ [t, isReadonly, isCreate],
);
const onOk = async current => {
