This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 38d2cdb437 [INLONG-10779][Dashboard] Data synchronization adds offline
synchronization configuration (#10782)
38d2cdb437 is described below
commit 38d2cdb4374951ee20bf2513d27daffff06faa7e
Author: kamianlaida <[email protected]>
AuthorDate: Tue Aug 20 10:52:45 2024 +0800
[INLONG-10779][Dashboard] Data synchronization adds offline synchronization
configuration (#10782)
Co-authored-by: Charles Zhang <[email protected]>
---
.../src/plugins/sync/common/SyncDefaultInfo.ts | 170 +++++++++++++++++++++
.../src/plugins/sync/common/SyncType.ts | 36 +++++
inlong-dashboard/src/ui/locales/cn.json | 28 +++-
inlong-dashboard/src/ui/locales/en.json | 26 ++++
.../src/ui/pages/SynchronizeDashboard/index.tsx | 19 ++-
.../src/ui/pages/SynchronizeDetail/Info/config.tsx | 77 +++++++++-
.../src/ui/pages/SynchronizeDetail/Info/index.tsx | 38 ++++-
7 files changed, 384 insertions(+), 10 deletions(-)
diff --git a/inlong-dashboard/src/plugins/sync/common/SyncDefaultInfo.ts
b/inlong-dashboard/src/plugins/sync/common/SyncDefaultInfo.ts
index 9e81013208..4e229a73f0 100644
--- a/inlong-dashboard/src/plugins/sync/common/SyncDefaultInfo.ts
+++ b/inlong-dashboard/src/plugins/sync/common/SyncDefaultInfo.ts
@@ -24,11 +24,18 @@ import i18n from '@/i18n';
import UserSelect from '@/ui/components/UserSelect';
import { genStatusTag, statusList } from './status';
import { timestampFormat } from '@/core/utils';
+import { DatePicker, TimePicker } from 'antd';
+import dayjs from 'dayjs';
+import { inlongGroupModeList } from '@/plugins/sync/common/SyncType';
+import { range } from 'lodash';
const { I18nMap, I18n } = DataWithBackend;
const { FieldList, FieldDecorator } = RenderRow;
const { ColumnList, ColumnDecorator } = RenderList;
+const format = 'HH:mm';
+const conventionalTimeFormat = 'YYYY-MM-DD HH:mm';
+
export class SyncDefaultInfo implements DataWithBackend, RenderRow, RenderList
{
static I18nMap = I18nMap;
static FieldList = FieldList;
@@ -87,7 +94,170 @@ export class SyncDefaultInfo implements DataWithBackend,
RenderRow, RenderList {
})
@I18n('meta.Synchronize.SinkMultipleEnable')
sinkMultipleEnable: boolean;
+ @FieldDecorator({
+ type: 'radio',
+ initialValue: 1,
+ rules: [{ required: true }],
+ props: {
+ options: [
+ {
+ label: i18n.t('meta.Synchronize.RealTime'),
+ value: 1,
+ },
+ {
+ label: i18n.t('meta.Synchronize.Offline'),
+ value: 2,
+ },
+ ],
+ },
+ })
+ @ColumnDecorator({
+ width: 200,
+ render: text => inlongGroupModeList?.filter(item => item.value ===
text)?.[0]?.label,
+ })
+ @I18n('meta.Synchronize.SyncType')
+ inlongGroupMode: Number;
+ @FieldDecorator({
+ type: 'radio',
+ initialValue: 0,
+ visible: values => values.inlongGroupMode === 2,
+ rules: [{ required: true }],
+ props: {
+ options: [
+ {
+ label: i18n.t('meta.Synchronize.Conventional'),
+ value: 0,
+ },
+ {
+ label: i18n.t('meta.Synchronize.Crontab'),
+ value: 1,
+ },
+ ],
+ },
+ })
+ @I18n('meta.Synchronize.ScheduleType')
+ scheduleType: Number;
+ @FieldDecorator({
+ visible: values => values.inlongGroupMode === 2 && values.scheduleType ===
0,
+ type: 'select',
+ initialValue: 'H',
+ name: 'scheduleUnit',
+ rules: [{ required: true }],
+ props: {
+ options: [
+ {
+ label: i18n.t('meta.Synchronize.ScheduleUnit.Year'),
+ value: 'Y',
+ },
+ {
+ label: i18n.t('meta.Synchronize.ScheduleUnit.Month'),
+ value: 'M',
+ },
+ {
+ label: i18n.t('meta.Synchronize.ScheduleUnit.Day'),
+ value: 'D',
+ },
+ {
+ label: i18n.t('meta.Synchronize.ScheduleUnit.Hours'),
+ value: 'H',
+ },
+ {
+ label: i18n.t('meta.Synchronize.ScheduleUnit.Minute'),
+ value: 'I',
+ },
+ {
+ label: i18n.t('meta.Synchronize.ScheduleUnit.Single'),
+ value: 'O',
+ },
+ ],
+ },
+ })
+ @I18n('meta.Synchronize.ScheduleUnit')
+ scheduleUnit: String;
+
+ @FieldDecorator({
+ type: 'input',
+ initialValue: 0,
+ rules: [{ required: true, pattern: new RegExp(/^[0-9]\d*$/, 'g') }],
+ visible: values => values.inlongGroupMode === 2 && values.scheduleType ===
0,
+ props: values => ({
+ suffix: values.scheduleUnit,
+ }),
+ })
+ @I18n('meta.Synchronize.ScheduleInterval')
+ scheduleInterval: number;
+
+ @FieldDecorator({
+ visible: values => values.inlongGroupMode === 2 && values.scheduleType ===
0,
+ type: TimePicker,
+ initialValue: dayjs('00:00', format),
+ name: 'delayTime',
+ rules: [{ required: true }],
+ props: {
+ format: format,
+ },
+ })
+ @I18n('meta.Synchronize.DelayTime')
+ delayTime: dayjs.Dayjs;
+
+ @FieldDecorator({
+ type: DatePicker.RangePicker,
+ props: values => ({
+ format: conventionalTimeFormat,
+ showTime: true,
+ disabledTime: (date: dayjs.Dayjs, type, info: { from?: dayjs.Dayjs }) =>
{
+ return {
+ disabledSeconds: () => range(0, 60),
+ };
+ },
+ }),
+ visible: values =>
+ values.inlongGroupMode === 2 && (values.scheduleType === 0 ||
values.scheduleType === 1),
+ rules: [{ required: true }],
+ })
+ @I18n('meta.Synchronize.ValidTime')
+ time: dayjs.Dayjs[];
+ @FieldDecorator({
+ visible: values => values.inlongGroupMode === 2 && values.scheduleType ===
1,
+ type: 'input',
+ rules: [{ required: true }],
+ props: {},
+ })
+ @I18n('meta.Synchronize.CronExpression')
+ crontabExpression: string;
+
+ @FieldDecorator({
+ type: 'radio',
+ visible: values => values.inlongGroupMode === 2,
+ initialValue: 0,
+ rules: [{ required: true }],
+ props: {
+ options: [
+ {
+ label: i18n.t('meta.Synchronize.Alone'),
+ value: 0,
+ },
+ {
+ label: i18n.t('meta.Synchronize.Self'),
+ value: 1,
+ },
+ {
+ label: i18n.t('meta.Synchronize.Parallel'),
+ value: 2,
+ },
+ ],
+ },
+ })
+ @I18n('meta.Synchronize.SelfDependence')
+ selfDepend: Number;
+ @FieldDecorator({
+ type: 'input',
+ rules: [{ required: true }],
+ visible: values => values.selfDepend === 2 && values.inlongGroupMode === 2,
+ })
+ @I18n('meta.Synchronize.TaskParallelism')
+ taskParallelism: string;
@FieldDecorator({
type: 'textarea',
props: {
diff --git a/inlong-dashboard/src/plugins/sync/common/SyncType.ts
b/inlong-dashboard/src/plugins/sync/common/SyncType.ts
new file mode 100644
index 0000000000..1c51e7e1d1
--- /dev/null
+++ b/inlong-dashboard/src/plugins/sync/common/SyncType.ts
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import i18n from '@/i18n';
+
+type TypeProp = {
+ label: string;
+ value: number;
+};
+
+export const inlongGroupModeList: TypeProp[] = [
+ {
+ label: i18n.t('meta.Synchronize.RealTimeSync'),
+ value: 1,
+ },
+ {
+ label: i18n.t('meta.Synchronize.OfflineSync'),
+ value: 2,
+ },
+];
diff --git a/inlong-dashboard/src/ui/locales/cn.json
b/inlong-dashboard/src/ui/locales/cn.json
index 59c69445c7..2c00ea1dc0 100644
--- a/inlong-dashboard/src/ui/locales/cn.json
+++ b/inlong-dashboard/src/ui/locales/cn.json
@@ -417,7 +417,33 @@
"meta.Synchronize.GroupIdHelp": "数据流组 ID 与数据流 ID 默认相同",
"meta.Synchronize.InlongGroupOwnersExtra": "责任人，可查看、修改数据同步信息",
"meta.Synchronize.GroupOwners": "责任人",
- "meta.Stream.StreamId": "数据流 ID",
+ "meta.Synchronize.SyncType": "同步类型",
+ "meta.Synchronize.RealTimeSync": "实时同步",
+ "meta.Synchronize.RealTime": "实时",
+ "meta.Synchronize.Offline": "离线",
+ "meta.Synchronize.OfflineSync": "离线同步",
+ "meta.Synchronize.SchedulingRules": "调度规则",
+ "meta.Synchronize.ScheduleType": "调度类型",
+ "meta.Synchronize.Conventional": "常规",
+ "meta.Synchronize.Crontab": "Crontab",
+ "meta.Synchronize.DelayTime": "延迟时间",
+ "meta.Synchronize.ScheduleUnit": "调度单位",
+ "meta.Synchronize.ScheduleUnit.Year": "年",
+ "meta.Synchronize.ScheduleUnit.Month": "月",
+ "meta.Synchronize.ScheduleUnit.Day": "天",
+ "meta.Synchronize.ScheduleUnit.Minute": "分钟",
+ "meta.Synchronize.ScheduleUnit.Hours": "小时",
+ "meta.Synchronize.ScheduleUnit.Single": "单次",
+ "meta.Synchronize.ScheduleInterval": "调度周期",
+ "meta.Synchronize.ValidTime": "有效时间",
+ "meta.Synchronize.CronExpression": "Crontab 表达式",
+ "meta.Synchronize.DependConfiguration": "依赖配置",
+ "meta.Synchronize.SelfDependence": "自身依赖",
+ "meta.Synchronize.Alone": "单实例运行",
+ "meta.Synchronize.Self": "自依赖",
+ "meta.Synchronize.Parallel": "并行",
+ "meta.Synchronize.TaskParallelism": "最大并行个数",
+ "meta.Stream.StreamId":"数据流 ID",
"meta.Stream.StreamIdRules": "只能包含英文字母、数字、点号(.)、中划线(-)、下划线(_)",
"meta.Stream.DataSeparator": "源数据字段分割符",
"meta.Stream.DataSeparator.Space": "空格",
diff --git a/inlong-dashboard/src/ui/locales/en.json
b/inlong-dashboard/src/ui/locales/en.json
index 82bb47c46b..7546f2e399 100644
--- a/inlong-dashboard/src/ui/locales/en.json
+++ b/inlong-dashboard/src/ui/locales/en.json
@@ -417,6 +417,32 @@
"meta.Synchronize.GroupIdHelp": "Group ID and stream ID are the same by
default",
"meta.Synchronize.InlongGroupOwnersExtra": "Can view, modify data
synchronization info",
"meta.Synchronize.GroupOwners": "Group owners",
+ "meta.Synchronize.SyncType": "Sync Type",
+ "meta.Synchronize.RealTimeSync": "RealTime Sync",
+ "meta.Synchronize.RealTime": "RealTime",
+ "meta.Synchronize.Offline": "Offline",
+ "meta.Synchronize.OfflineSync": "Offline Sync",
+ "meta.Synchronize.SchedulingRules": "Scheduling Rules",
+ "meta.Synchronize.ScheduleType": "Schedule Type",
+ "meta.Synchronize.Conventional": "Conventional",
+ "meta.Synchronize.Crontab": "Crontab",
+ "meta.Synchronize.DelayTime": "Delay Time",
+ "meta.Synchronize.ScheduleUnit": "Schedule Unit",
+ "meta.Synchronize.ScheduleUnit.Year": "Year",
+ "meta.Synchronize.ScheduleUnit.Month": "Month",
+ "meta.Synchronize.ScheduleUnit.Day": "Day",
+ "meta.Synchronize.ScheduleUnit.Minute": "Minute",
+ "meta.Synchronize.ScheduleUnit.Hours": "Hours",
+ "meta.Synchronize.ScheduleUnit.Single": "Single",
+ "meta.Synchronize.ScheduleInterval": "Schedule Interval",
+ "meta.Synchronize.ValidTime": "Valid Time",
+ "meta.Synchronize.CronExpression": "Cron Expression",
+ "meta.Synchronize.DependConfiguration": "Depend Configuration",
+ "meta.Synchronize.SelfDependence": "Self Dependence",
+ "meta.Synchronize.Alone": "Alone",
+ "meta.Synchronize.Self": "Self",
+ "meta.Synchronize.Parallel": "Parallel",
+ "meta.Synchronize.TaskParallelism": "Task Parallelism",
"meta.Stream.StreamId": "Stream id",
"meta.Stream.StreamIdRules": "Only English letters, numbers, dots(.),
minus(-), and underscores(_)",
"meta.Stream.DataSeparator": "Source data separator",
diff --git a/inlong-dashboard/src/ui/pages/SynchronizeDashboard/index.tsx
b/inlong-dashboard/src/ui/pages/SynchronizeDashboard/index.tsx
index 2dfad1d5e4..b096f9957f 100644
--- a/inlong-dashboard/src/ui/pages/SynchronizeDashboard/index.tsx
+++ b/inlong-dashboard/src/ui/pages/SynchronizeDashboard/index.tsx
@@ -30,6 +30,7 @@ import { GroupLogs } from '@/ui/components/GroupLogs';
import { dashCardList, useColumns } from './config';
import { statusList } from '@/plugins/groups/common/status';
import { useDefaultMeta } from '@/plugins';
+import { inlongGroupModeList } from '@/plugins/sync/common/SyncType';
const Comp: React.FC = () => {
const { options: groups } = useDefaultMeta('group');
@@ -50,7 +51,7 @@ const Comp: React.FC = () => {
const { data: summary = {} } = useRequest({
url: '/group/countByStatus',
params: {
- inlongGroupMode: 1,
+ inlongGroupMode: options.inlongGroupMode,
},
});
@@ -171,6 +172,17 @@ const Comp: React.FC = () => {
dropdownMatchSelectWidth: false,
},
},
+ {
+ type: 'select',
+ name: 'inlongGroupMode',
+ label: i18n.t('meta.Synchronize.SyncType'),
+ initialValue: defaultValues.inlongGroupMode,
+ props: {
+ allowClear: true,
+ options: inlongGroupModeList,
+ dropdownMatchSelectWidth: false,
+ },
+ },
],
[groups],
);
@@ -196,7 +208,10 @@ const Comp: React.FC = () => {
table={{
columns,
rowKey: 'id',
- dataSource: data?.list,
+ dataSource: data?.list?.map(item => ({
+ ...item,
+ inlongGroupMode: options.inlongGroupMode,
+ })),
pagination,
loading,
onChange,
diff --git a/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/config.tsx
b/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/config.tsx
index 1b9e83fccf..78aac3df65 100644
--- a/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/config.tsx
+++ b/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/config.tsx
@@ -17,12 +17,14 @@
* under the License.
*/
-import React, { useMemo } from 'react';
+import React, { useCallback, useMemo } from 'react';
import { Divider } from 'antd';
import i18n from '@/i18n';
import { useLoadMeta, SyncMetaType } from '@/plugins';
import { excludeObjectArray } from '@/core/utils';
-
+import dayjs from 'dayjs';
+import { range } from 'lodash';
+const conventionalTimeFormat = 'YYYY-MM-DD HH:mm';
export const useFormContent = ({ mqType, editing, isCreate, isUpdate }) => {
const { Entity } = useLoadMeta<SyncMetaType>('sync', mqType);
@@ -48,6 +50,21 @@ export const useFormContent = ({ mqType, editing, isCreate,
isUpdate }) => {
})
: fields.map(item => {
const t = transType(editing, item);
+ if (item.name === 'time' || item.name === 'delayTime') {
+ return {
+ ...item,
+ props: values => ({
+ format: conventionalTimeFormat,
+ showTime: true,
+ disabledTime: (date: dayjs.Dayjs, type, info: { from?:
dayjs.Dayjs }) => {
+ return {
+ disabledSeconds: () => range(0, 60),
+ };
+ },
+ disabled: !editing,
+ }),
+ };
+ }
return {
...item,
type: t,
@@ -62,13 +79,58 @@ export const useFormContent = ({ mqType, editing, isCreate,
isUpdate }) => {
rules: t === 'text' ? undefined : item.rules,
};
});
+ const isScKey = useCallback(
+ formName => {
+ const defaultGroupKeysI18nMap = Entity?.I18nMap || {};
+ return (
+ !defaultGroupKeysI18nMap[formName] ||
+ [
+ 'scheduleType',
+ 'time',
+ 'crontabExpression',
+ 'scheduledCycle',
+ 'scheduleUnit',
+ 'scheduleInterval',
+ 'delayTime',
+ ].includes(formName)
+ );
+ },
+ [Entity?.I18nMap],
+ );
+
+ const isSrKey = useCallback(
+ formName => {
+ const defaultGroupKeysI18nMap = Entity?.I18nMap || {};
+ return (
+ !defaultGroupKeysI18nMap[formName] || ['selfDepend',
'taskParallelism'].includes(formName)
+ );
+ },
+ [Entity?.I18nMap],
+ );
+
+ const groupFormContent = formContent.filter(item => !isScKey(item.name));
+ const baseFormContent = groupFormContent.filter(item => !isSrKey(item.name));
+ const scFormContent = formContent.filter(item => isScKey(item.name));
+ const srFormContent = formContent.filter(item => isSrKey(item.name));
return [
{
type: <Divider
orientation="left">{i18n.t('pages.GroupDetail.Info.Basic')}</Divider>,
col: 24,
},
- ...formContent,
+ ...baseFormContent,
+ {
+ visible: values => values.inlongGroupMode === 2,
+ type: <Divider
orientation="left">{i18n.t('meta.Synchronize.SchedulingRules')}</Divider>,
+ col: 24,
+ },
+ ...scFormContent,
+ {
+ visible: values => values.inlongGroupMode === 2,
+ type: <Divider
orientation="left">{i18n.t('meta.Synchronize.DependConfiguration')}</Divider>,
+ col: 24,
+ },
+ ...srFormContent,
];
};
@@ -87,6 +149,12 @@ function transType(editing: boolean, conf) {
'ttl',
'retentionTime',
'retentionSize',
+ 'scheduleType',
+ 'scheduleUnit',
+ 'scheduleInterval',
+ 'crontabExpression',
+ 'selfDepend',
+ 'taskParallelism',
],
as: 'text',
active: !editing,
@@ -100,6 +168,9 @@ function transType(editing: boolean, conf) {
const item = map.get(conf.name);
return item.active ? item.as : conf.type;
}
+ if (conf.name === 'time' || conf.name === 'delayTime') {
+ return conf.type;
+ }
return 'text';
}
diff --git a/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/index.tsx
b/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/index.tsx
index e6251dcad4..28784c7826 100644
--- a/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/index.tsx
+++ b/inlong-dashboard/src/ui/pages/SynchronizeDetail/Info/index.tsx
@@ -27,13 +27,14 @@ import request from '@/core/utils/request';
import { useFormContent } from './config';
import { CommonInterface } from '../common';
import { State } from '@/core/stores';
+import dayjs from 'dayjs';
type Props = CommonInterface;
const Comp = ({ inlongGroupId, inlongStreamId, readonly, isCreate }: Props,
ref) => {
const { t } = useTranslation();
const [editing, { setTrue, setFalse }] = useBoolean(isCreate);
-
+ const conventionalTimeFormat = 'YYYY-MM-DD HH:mm';
const { defaultValue } = useDefaultMeta('sync');
const { userName } = useSelector<State, State>(state => state);
@@ -46,6 +47,13 @@ const Comp = ({ inlongGroupId, inlongStreamId, readonly,
isCreate }: Props, ref)
return !!inlongGroupId;
}, [inlongGroupId]);
+ useEffect(() => {
+ if (isCreate) {
+ form.setFieldValue('scheduleType', 0);
+ form.setFieldValue('scheduleUnit', 'H');
+ }
+ }, [form, isCreate]);
+
const isUpdateStream = useMemo(() => {
return !!inlongStreamId;
}, [inlongStreamId]);
@@ -56,6 +64,11 @@ const Comp = ({ inlongGroupId, inlongStreamId, readonly,
isCreate }: Props, ref)
formatResult: data => ({
...data,
inCharges: data.inCharges.split(','),
+ time: [
+ dayjs(dayjs(data?.startTime), conventionalTimeFormat),
+ dayjs(dayjs(data?.endTime), conventionalTimeFormat),
+ ],
+ delayTime: convertMinutesToDelayTime(data.delayTime),
}),
onSuccess: data => {
setMqType(data.mqType);
@@ -77,6 +90,16 @@ const Comp = ({ inlongGroupId, inlongStreamId, readonly,
isCreate }: Props, ref)
},
},
);
+ const convertTimeToMinutes = timeString => {
+ const time = dayjs(timeString, 'HH:mm');
+ return time.hour() * 60 + time.minute();
+ };
+
+ const convertMinutesToDelayTime = totalMinutes => {
+ const hours = Math.floor(totalMinutes / 60);
+ const minutes = totalMinutes % 60;
+ return dayjs().hour(hours).minute(minutes);
+ };
const { data: streamData, run: getDataStream } = useRequest(
{
url: '/stream/list',
@@ -99,13 +122,20 @@ const Comp = ({ inlongGroupId, inlongStreamId, readonly,
isCreate }: Props, ref)
const onOk = async () => {
const values = await form.validateFields();
- const submitData = {
+ let submitData = {
...values,
version: data?.version,
inCharges: values.inCharges?.join(','),
- inlongGroupMode: 1,
};
-
+ if (values.inlongGroupMode === 2) {
+ submitData = {
+ ...submitData,
+ delayTime: convertTimeToMinutes(values?.delayTime?.format('HH:mm')),
+ startTime:
dayjs(values?.time?.[0]?.format(conventionalTimeFormat)).valueOf(),
+ endTime:
dayjs(values?.time?.[1]?.format(conventionalTimeFormat)).valueOf(),
+ };
+ }
+ delete submitData.time;
const submitDataStream = {
inlongGroupId: values.inlongGroupId,
inlongStreamId: values.inlongGroupId,
