This is an automated email from the ASF dual-hosted git repository.
leezng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 7f4eb56f55 [INLONG-8046][Dashboard] Support batch import of sink
fields (#8053)
7f4eb56f55 is described below
commit 7f4eb56f55c9bfdcfe8dd6e48413627262525b9b
Author: feat <[email protected]>
AuthorDate: Wed Jun 14 10:16:27 2023 +0800
[INLONG-8046][Dashboard] Support batch import of sink fields (#8053)
---
.../src/plugins/sinks/defaults/ClickHouse.ts | 2 +
.../src/plugins/sinks/defaults/Doris.ts | 2 +
.../src/plugins/sinks/defaults/Elasticsearch.ts | 2 +
.../src/plugins/sinks/defaults/Greenplum.ts | 2 +
.../src/plugins/sinks/defaults/HBase.ts | 2 +
.../src/plugins/sinks/defaults/Hive.ts | 2 +
.../src/plugins/sinks/defaults/Hudi.ts | 2 +
.../src/plugins/sinks/defaults/Iceberg.ts | 2 +
.../src/plugins/sinks/defaults/Kudu.ts | 2 +
.../src/plugins/sinks/defaults/MySQL.ts | 2 +
.../src/plugins/sinks/defaults/Oracle.ts | 2 +
.../src/plugins/sinks/defaults/PostgreSQL.ts | 2 +
.../src/plugins/sinks/defaults/Redis.ts | 2 +
.../src/plugins/sinks/defaults/SQLServer.ts | 2 +
.../src/plugins/sinks/defaults/StarRocks.ts | 2 +
.../src/plugins/sinks/defaults/TDSQLPostgreSQL.ts | 2 +
.../src/ui/components/EditableTable/index.tsx | 65 +++++++++++++++++-----
17 files changed, 84 insertions(+), 13 deletions(-)
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/ClickHouse.ts
b/inlong-dashboard/src/plugins/sinks/defaults/ClickHouse.ts
index 8b082ad173..780f61da5b 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/ClickHouse.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/ClickHouse.ts
@@ -321,6 +321,8 @@ export default class ClickHouseSink
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/Doris.ts
b/inlong-dashboard/src/plugins/sinks/defaults/Doris.ts
index 514934a8a1..cf2b9c8136 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/Doris.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/Doris.ts
@@ -177,6 +177,8 @@ export default class DorisSink extends SinkInfo implements
DataWithBackend, Rend
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/Elasticsearch.ts
b/inlong-dashboard/src/plugins/sinks/defaults/Elasticsearch.ts
index e2a49cfae0..5d19d7dce9 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/Elasticsearch.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/Elasticsearch.ts
@@ -166,6 +166,8 @@ export default class ElasticsearchSink
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/Greenplum.ts
b/inlong-dashboard/src/plugins/sinks/defaults/Greenplum.ts
index a156585e29..dde187f908 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/Greenplum.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/Greenplum.ts
@@ -150,6 +150,8 @@ export default class GreenplumSink
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/HBase.ts
b/inlong-dashboard/src/plugins/sinks/defaults/HBase.ts
index 8de96c1f39..fa9b71e5d0 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/HBase.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/HBase.ts
@@ -146,6 +146,8 @@ export default class HBaseSink extends SinkInfo implements
DataWithBackend, Rend
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertKey: 'fieldName',
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/Hive.ts
b/inlong-dashboard/src/plugins/sinks/defaults/Hive.ts
index 004cd7d883..99548215cc 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/Hive.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/Hive.ts
@@ -244,6 +244,8 @@ export default class HiveSink extends SinkInfo implements
DataWithBackend, Rende
size: 'small',
columns: getFieldListColumns(values),
canDelete: ![110, 130].includes(values?.status),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/Hudi.ts
b/inlong-dashboard/src/plugins/sinks/defaults/Hudi.ts
index b54b7a3803..f0b2056322 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/Hudi.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/Hudi.ts
@@ -244,6 +244,8 @@ export default class HudiSink extends SinkInfo implements
DataWithBackend, Rende
type: EditableTable,
props: values => ({
size: 'small',
+ canBatchAdd: true,
+ upsetByFieldKey: true,
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
}),
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/Iceberg.ts
b/inlong-dashboard/src/plugins/sinks/defaults/Iceberg.ts
index 7f23b5b71d..2a1b29a146 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/Iceberg.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/Iceberg.ts
@@ -247,6 +247,8 @@ export default class IcebergSink
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/Kudu.ts
b/inlong-dashboard/src/plugins/sinks/defaults/Kudu.ts
index d895796e19..5ab325174b 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/Kudu.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/Kudu.ts
@@ -188,6 +188,8 @@ export default class KuduSink extends SinkInfo implements
DataWithBackend, Rende
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/MySQL.ts
b/inlong-dashboard/src/plugins/sinks/defaults/MySQL.ts
index 54780f8bd0..5d62467345 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/MySQL.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/MySQL.ts
@@ -136,6 +136,8 @@ export default class HiveSink extends SinkInfo implements
DataWithBackend, Rende
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/Oracle.ts
b/inlong-dashboard/src/plugins/sinks/defaults/Oracle.ts
index bbd08d8ed6..4f7c9baa1d 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/Oracle.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/Oracle.ts
@@ -143,6 +143,8 @@ export default class OracleSink extends SinkInfo implements
DataWithBackend, Ren
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/PostgreSQL.ts
b/inlong-dashboard/src/plugins/sinks/defaults/PostgreSQL.ts
index 5c6bb76613..6159eda722 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/PostgreSQL.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/PostgreSQL.ts
@@ -158,6 +158,8 @@ export default class HiveSink extends SinkInfo implements
DataWithBackend, Rende
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/Redis.ts
b/inlong-dashboard/src/plugins/sinks/defaults/Redis.ts
index 6b95d4a488..0d89d644b8 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/Redis.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/Redis.ts
@@ -276,6 +276,8 @@ export default class RedisSink extends SinkInfo implements
DataWithBackend, Rend
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/SQLServer.ts
b/inlong-dashboard/src/plugins/sinks/defaults/SQLServer.ts
index 958992a797..f5e696a2fc 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/SQLServer.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/SQLServer.ts
@@ -195,6 +195,8 @@ export default class SqlServerSink
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/StarRocks.ts
b/inlong-dashboard/src/plugins/sinks/defaults/StarRocks.ts
index 6a20aeb7bc..eaa754d193 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/StarRocks.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/StarRocks.ts
@@ -168,6 +168,8 @@ export default class StarRocksSink
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/plugins/sinks/defaults/TDSQLPostgreSQL.ts
b/inlong-dashboard/src/plugins/sinks/defaults/TDSQLPostgreSQL.ts
index 785922467a..244b854dab 100644
--- a/inlong-dashboard/src/plugins/sinks/defaults/TDSQLPostgreSQL.ts
+++ b/inlong-dashboard/src/plugins/sinks/defaults/TDSQLPostgreSQL.ts
@@ -153,6 +153,8 @@ export default class TDSQLPostgreSQLSink
size: 'small',
editing: ![110, 130].includes(values?.status),
columns: getFieldListColumns(values),
+ canBatchAdd: true,
+ upsertByFieldKey: true,
}),
})
sinkFieldList: Record<string, unknown>[];
diff --git a/inlong-dashboard/src/ui/components/EditableTable/index.tsx
b/inlong-dashboard/src/ui/components/EditableTable/index.tsx
index 262a0d208b..2eda1d9c94 100644
--- a/inlong-dashboard/src/ui/components/EditableTable/index.tsx
+++ b/inlong-dashboard/src/ui/components/EditableTable/index.tsx
@@ -69,6 +69,10 @@ export interface EditableTableProps
// Can add a new line? Default: true.
canAdd?: boolean;
canBatchAdd?: boolean;
+ upsetByFieldKey?: boolean;
+ fieldNameKey?: string;
+ fieldTypeKey?: string;
+ fieldCommentKey?: string;
}
const getRowInitialValue = (columns: EditableTableProps['columns']) =>
@@ -106,6 +110,10 @@ const EditableTable = ({
canDelete = true,
canAdd = true,
canBatchAdd = false,
+ upsetByFieldKey = false,
+ fieldNameKey = 'fieldName',
+ fieldTypeKey = 'fieldType',
+ fieldCommentKey = 'fieldComment',
...rest
}: EditableTableProps) => {
if (!id) {
@@ -169,23 +177,54 @@ const EditableTable = ({
};
const onAppendByParseField = (fields: RowType[]) => {
- const newRecord: RecordType[] = fields?.map((field: RowType) => ({
- _etid: Math.random().toString(),
- ...field,
- }));
- const newData = data.concat(newRecord);
- setData(newData);
- triggerChange(newData);
+ // append empty row if upsertKey not null
+ if (upsetByFieldKey) {
+ let index: number = 0;
+ data.forEach(item => {
+ if (item[fieldNameKey] === '') {
+ item[fieldNameKey] = fields[index]['fieldName'];
+ item[fieldTypeKey] = fields[index]['fieldType'];
+ item[fieldCommentKey] = fields[index]['fieldComment'];
+ index++;
+ }
+ });
+
+ setData(data);
+ triggerChange(data);
+ } else {
+ const newRecord: RecordType[] = fields?.map((field: RowType) => ({
+ _etid: Math.random().toString(),
+ ...field,
+ }));
+ const newData = data.concat(newRecord);
+
+ setData(newData);
+ triggerChange(newData);
+ }
};
const onOverrideByParseField = (fields: RowType[]) => {
- const newData = fields?.map(field => ({
- _etid: Math.random().toString(),
- ...field,
- }));
+ // append empty row if upsertKey not null
+ if (upsetByFieldKey) {
+ let index: number = 0;
+ data.forEach(item => {
+ item[fieldNameKey] = fields[index]['fieldName'];
+ item[fieldTypeKey] = fields[index]['fieldType'];
+ item[fieldCommentKey] = fields[index]['fieldComment'];
+ index++;
+ });
- setData(newData);
- triggerChange(newData);
+ setData(data);
+ triggerChange(data);
+ } else {
+ const newData = fields?.map(field => ({
+ _etid: Math.random().toString(),
+ ...field,
+ }));
+
+ setData(newData);
+ triggerChange(newData);
+ }
};
const onTextChange = (object: Record<string, unknown>, { _etid }:
RecordType) => {
