This is an automated email from the ASF dual-hosted git repository.
leezng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new da1a03b1e [INLONG-6786][Dashboard] Supoort Apache Hudi sink management
(#6791)
da1a03b1e is described below
commit da1a03b1e628da3a64fe2ab69ba45777d978b77d
Author: averyzhang <[email protected]>
AuthorDate: Fri Dec 9 10:05:47 2022 +0800
[INLONG-6786][Dashboard] Supoort Apache Hudi sink management (#6791)
---
inlong-dashboard/src/locales/cn.json | 17 +
inlong-dashboard/src/locales/en.json | 16 +
inlong-dashboard/src/metas/sinks/defaults/Hudi.ts | 398 +++++++++++++++++++++
inlong-dashboard/src/metas/sinks/defaults/index.ts | 5 +
4 files changed, 436 insertions(+)
diff --git a/inlong-dashboard/src/locales/cn.json
b/inlong-dashboard/src/locales/cn.json
index e6e264360..16ba0ba58 100644
--- a/inlong-dashboard/src/locales/cn.json
+++ b/inlong-dashboard/src/locales/cn.json
@@ -178,6 +178,23 @@
"meta.Sinks.Iceberg.FieldDescription": "字段描述",
"meta.Sinks.Iceberg.PartitionStrategy": "分区策略",
"meta.Sinks.Iceberg.DataNodeName": "数据节点",
+ "meta.Sinks.Hudi.DbName": "DB名称",
+ "meta.Sinks.Hudi.TableName": "表名称",
+ "meta.Sinks.Hudi.Warehouse": "仓库路径",
+ "meta.Sinks.Hudi.FileFormat": "⽂件格式",
+ "meta.Sinks.Hudi.Description": "表描述",
+ "meta.Sinks.Hudi.ExtList": "属性",
+ "meta.Sinks.Hudi.DataConsistency": "数据一致性",
+ "meta.Sinks.Hudi.FieldName": "字段名",
+ "meta.Sinks.Hudi.FieldNameRule": "以英文字母或下划线开头，只能包含英文字母、数字、下划线",
+ "meta.Sinks.Hudi.FieldType": "字段类型",
+ "meta.Sinks.Hudi.FieldDescription": "字段描述",
+ "meta.Sinks.Hudi.PrimaryKey": "主键",
+ "meta.Sinks.Hudi.PartitionFieldList": "分区字段",
+ "meta.Sinks.Hudi.PrimaryKeyHelper": "主键字段，以逗号(,)分割",
+ "meta.Sinks.Hudi.PartitionFieldListHelp": "字段类型若为timestamp，则必须设置此字段值的格式，支持
MICROSECONDS，MILLISECONDS，SECONDS，SQL，ISO_8601，以及自定义，比如：yyyy-MM-dd HH:mm:ss 等",
+ "meta.Sinks.Hudi.FieldFormat": "字段格式",
+ "meta.Sinks.Hudi.ExtListHelper": "hudi表的DDL属性需带前缀'ddl.'",
"meta.Sinks.Greenplum.TableName": "表名称",
"meta.Sinks.Greenplum.PrimaryKey": "主键",
"meta.Sinks.Greenplum.FieldName": "字段名",
diff --git a/inlong-dashboard/src/locales/en.json
b/inlong-dashboard/src/locales/en.json
index e20e88c85..bbe1abac3 100644
--- a/inlong-dashboard/src/locales/en.json
+++ b/inlong-dashboard/src/locales/en.json
@@ -178,6 +178,22 @@
"meta.Sinks.Iceberg.FieldDescription": "FieldDescription",
"meta.Sinks.Iceberg.PartitionStrategy": "PartitionStrategy",
"meta.Sinks.Iceberg.DataNodeName": "DataNode",
+ "meta.Sinks.Hudi.DbName": "DbName",
+ "meta.Sinks.Hudi.TableName": "TableName",
+ "meta.Sinks.Hudi.Warehouse": "Warehouse",
+ "meta.Sinks.Hudi.FileFormat": "FileFormat",
+ "meta.Sinks.Hudi.Description": "Description",
+ "meta.Sinks.Hudi.ExtList": "ExtList",
+ "meta.Sinks.Hudi.DataConsistency": "DataConsistency",
+ "meta.Sinks.Hudi.FieldName": "FieldName",
+ "meta.Sinks.Hudi.FieldNameRule": "At the beginning of English letters or
underscore, only English letters, numbers, and underscores",
+ "meta.Sinks.Hudi.FieldType": "FieldType",
+ "meta.Sinks.Hudi.FieldDescription": "FieldDescription",
+ "meta.Sinks.Hudi.PrimaryKey": "PrimaryKey",
+ "meta.Sinks.Hudi.PrimaryKeyHelper": "The Primary key fields, separated by
commas (,)",
+ "meta.Sinks.Hudi.PartitionFieldList": "PartitionFieldList",
+ "meta.Sinks.Hudi.PartitionFieldListHelp": "If the field type is timestamp,
you must set the format of the field value, support MICROSECONDS, MILLISECONDS,
SECONDS, SQL, ISO_8601, and custom, such as: yyyy-MM-dd HH:mm:ss, etc.",
+ "meta.Sinks.Hudi.ExtListHelper": "The DDL attribute of the hudi table needs
to be prefixed with 'ddl.'",
"meta.Sinks.Greenplum.TableName": "TableName",
"meta.Sinks.Greenplum.PrimaryKey": "PrimaryKey",
"meta.Sinks.Greenplum.FieldName": "FieldName",
diff --git a/inlong-dashboard/src/metas/sinks/defaults/Hudi.ts
b/inlong-dashboard/src/metas/sinks/defaults/Hudi.ts
new file mode 100644
index 000000000..ad4648ac2
--- /dev/null
+++ b/inlong-dashboard/src/metas/sinks/defaults/Hudi.ts
@@ -0,0 +1,398 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { DataWithBackend } from '@/metas/DataWithBackend';
+import { RenderRow } from '@/metas/RenderRow';
+import { RenderList } from '@/metas/RenderList';
+import i18n from '@/i18n';
+import EditableTable from '@/components/EditableTable';
+import { sourceFields } from '../common/sourceFields';
+import { SinkInfo } from '../common/SinkInfo';
+
+const { I18n } = DataWithBackend;
+const { FieldDecorator } = RenderRow;
+const { ColumnDecorator } = RenderList;
+
+const hudiFieldTypes = [
+ 'int',
+ 'long',
+ 'string',
+ 'float',
+ 'double',
+ 'date',
+ 'timestamp',
+ 'time',
+ 'boolean',
+ 'decimal',
+ 'timestamptz',
+ 'binary',
+ 'fixed',
+ 'uuid',
+].map(item => ({
+ label: item,
+ value: item,
+}));
+
+const matchPartitionStrategies = fieldType => {
+ const data = [
+ {
+ label: 'None',
+ value: 'None',
+ disabled: false,
+ },
+ {
+ label: 'Identity',
+ value: 'Identity',
+ disabled: false,
+ },
+ {
+ label: 'Year',
+ value: 'Year',
+ disabled: !['timestamp', 'date'].includes(fieldType),
+ },
+ {
+ label: 'Month',
+ value: 'Month',
+ disabled: !['timestamp', 'date'].includes(fieldType),
+ },
+ {
+ label: 'Day',
+ value: 'Day',
+ disabled: !['timestamp', 'date'].includes(fieldType),
+ },
+ {
+ label: 'Hour',
+ value: 'Hour',
+ disabled: fieldType !== 'timestamp',
+ },
+ {
+ label: 'Bucket',
+ value: 'Bucket',
+ disabled: ![
+ 'string',
+ 'boolean',
+ 'short',
+ 'int',
+ 'long',
+ 'float',
+ 'double',
+ 'decimal',
+ ].includes(fieldType),
+ },
+ {
+ label: 'Truncate',
+ value: 'Truncate',
+ disabled: !['string', 'int', 'long', 'binary',
'decimal'].includes(fieldType),
+ },
+ ];
+
+ return data.filter(item => !item.disabled);
+};
+
+export default class HudiSink extends SinkInfo implements DataWithBackend,
RenderRow, RenderList {
+ @FieldDecorator({
+ type: 'input',
+ rules: [{ required: true }],
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sinks.Hudi.DbName')
+ dbName: string;
+
+ @FieldDecorator({
+ type: 'input',
+ rules: [{ required: true }],
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sinks.Hudi.TableName')
+ tableName: string;
+
+ @FieldDecorator({
+ type: 'radio',
+ rules: [{ required: true }],
+ initialValue: 1,
+ tooltip: i18n.t('meta.Sinks.EnableCreateResourceHelp'),
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ options: [
+ {
+ label: i18n.t('basic.Yes'),
+ value: 1,
+ },
+ {
+ label: i18n.t('basic.No'),
+ value: 0,
+ },
+ ],
+ }),
+ })
+ @I18n('meta.Sinks.EnableCreateResource')
+ enableCreateResource: number;
+
+ @FieldDecorator({
+ type: 'input',
+ rules: [{ required: true }],
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ placeholder: 'thrift://127.0.0.1:9083',
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('Catalog URI')
+ catalogUri: string;
+
+ @FieldDecorator({
+ type: 'input',
+ rules: [{ required: true }],
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ placeholder: 'hdfs://127.0.0.1:9000/user/hudi/warehouse',
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sinks.Hudi.Warehouse')
+ warehouse: string;
+
+ @FieldDecorator({
+ type: 'select',
+ rules: [{ required: true }],
+ initialValue: 'Parquet',
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ options: [
+ {
+ label: 'Parquet',
+ value: 'Parquet',
+ },
+ // {
+ // label: 'Orc',
+ // value: 'Orc',
+ // },
+ // {
+ // label: 'Avro',
+ // value: 'Avro',
+ // },
+ ],
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sinks.Hudi.FileFormat')
+ fileFormat: string;
+
+ @FieldDecorator({
+ type: EditableTable,
+ rules: [{ required: false }],
+ initialValue: [],
+ tooltip: i18n.t('meta.Sinks.Hudi.ExtListHelper'),
+ props: values => ({
+ size: 'small',
+ columns: [
+ {
+ title: 'Key',
+ dataIndex: 'keyName',
+ props: {
+ disabled: [110, 130].includes(values?.status),
+ },
+ },
+ {
+ title: 'Value',
+ dataIndex: 'keyValue',
+ props: {
+ disabled: [110, 130].includes(values?.status),
+ },
+ },
+ ],
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sinks.Hudi.ExtList')
+ extList: string;
+
+ @FieldDecorator({
+ type: 'select',
+ rules: [{ required: true }],
+ initialValue: 'EXACTLY_ONCE',
+ isPro: true,
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ options: [
+ {
+ label: 'EXACTLY_ONCE',
+ value: 'EXACTLY_ONCE',
+ },
+ {
+ label: 'AT_LEAST_ONCE',
+ value: 'AT_LEAST_ONCE',
+ },
+ ],
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sinks.Hudi.DataConsistency')
+ dataConsistency: string;
+
+ @FieldDecorator({
+ type: EditableTable,
+ props: values => ({
+ size: 'small',
+ editing: ![110, 130].includes(values?.status),
+ columns: getFieldListColumns(values),
+ }),
+ })
+ sinkFieldList: Record<string, unknown>[];
+
+ @FieldDecorator({
+ type: EditableTable,
+ tooltip: i18n.t('meta.Sinks.Hudi.PartitionFieldListHelp'),
+ col: 24,
+ props: {
+ size: 'small',
+ required: false,
+ columns: [
+ {
+ title: i18n.t('meta.Sinks.Hudi.FieldName'),
+ dataIndex: 'fieldName',
+ rules: [{ required: true }],
+ },
+ {
+ title: i18n.t('meta.Sinks.Hudi.FieldType'),
+ dataIndex: 'fieldType',
+ type: 'select',
+ initialValue: 'string',
+ props: {
+ options: ['string', 'timestamp'].map(item => ({
+ label: item,
+ value: item,
+ })),
+ },
+ },
+ {
+ title: i18n.t('meta.Sinks.Hudi.FieldFormat'),
+ dataIndex: 'fieldFormat',
+ type: 'autocomplete',
+ props: {
+ options: ['MICROSECONDS', 'MILLISECONDS', 'SECONDS', 'SQL',
'ISO_8601'].map(item => ({
+ label: item,
+ value: item,
+ })),
+ },
+ rules: [{ required: true }],
+ visible: (text, record) => record.fieldType === 'timestamp',
+ },
+ ],
+ },
+ })
+ @I18n('meta.Sinks.Hudi.PartitionFieldList')
+ partitionFieldList: Record<string, unknown>[];
+
+ @FieldDecorator({
+ type: 'input',
+ tooltip: i18n.t('meta.Sinks.Hudi.PrimaryKeyHelper'),
+ rules: [{ required: true }],
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sinks.Hudi.PrimaryKey')
+ primaryKey: string;
+}
+
+const getFieldListColumns = sinkValues => {
+ return [
+ ...sourceFields,
+ {
+ title: `Hudi ${i18n.t('meta.Sinks.Hudi.FieldName')}`,
+ width: 110,
+ dataIndex: 'fieldName',
+ rules: [
+ { required: true },
+ {
+ pattern: /^[a-zA-Z_][a-zA-Z0-9_]*$/,
+ message: i18n.t('meta.Sinks.Hudi.FieldNameRule'),
+ },
+ ],
+ props: (text, record, idx, isNew) => ({
+ disabled: [110, 130].includes(sinkValues?.status as number) && !isNew,
+ }),
+ },
+ {
+ title: `Hudi ${i18n.t('meta.Sinks.Hudi.FieldType')}`,
+ dataIndex: 'fieldType',
+ width: 130,
+ initialValue: hudiFieldTypes[0].value,
+ type: 'select',
+ rules: [{ required: true }],
+ props: (text, record, idx, isNew) => ({
+ options: hudiFieldTypes,
+ onChange: value => {
+ const partitionStrategies = matchPartitionStrategies(value);
+ if (partitionStrategies.every(item => item.value !==
record.partitionStrategy)) {
+ return {
+ partitionStrategy: partitionStrategies[0].value,
+ };
+ }
+ },
+ disabled: [110, 130].includes(sinkValues?.status as number) && !isNew,
+ }),
+ },
+ {
+ title: 'Length',
+ dataIndex: 'fieldLength',
+ type: 'inputnumber',
+ props: {
+ min: 0,
+ },
+ initialValue: 1,
+ rules: [{ type: 'number', required: true }],
+ visible: (text, record) => record.fieldType === 'fixed',
+ },
+ {
+ title: 'Precision',
+ dataIndex: 'fieldPrecision',
+ type: 'inputnumber',
+ props: {
+ min: 0,
+ },
+ initialValue: 1,
+ rules: [{ type: 'number', required: true }],
+ visible: (text, record) => record.fieldType === 'decimal',
+ },
+ {
+ title: 'Scale',
+ dataIndex: 'fieldScale',
+ type: 'inputnumber',
+ props: {
+ min: 0,
+ },
+ initialValue: 1,
+ rules: [{ type: 'number', required: true }],
+ visible: (text, record) => record.fieldType === 'decimal',
+ },
+ {
+ title: i18n.t('meta.Sinks.Hudi.FieldDescription'),
+ dataIndex: 'fieldComment',
+ },
+ ];
+};
diff --git a/inlong-dashboard/src/metas/sinks/defaults/index.ts
b/inlong-dashboard/src/metas/sinks/defaults/index.ts
index 700013573..3ea2ef16d 100644
--- a/inlong-dashboard/src/metas/sinks/defaults/index.ts
+++ b/inlong-dashboard/src/metas/sinks/defaults/index.ts
@@ -61,6 +61,11 @@ export const allDefaultSinks:
MetaExportWithBackendList<SinkMetaType> = [
value: 'ICEBERG',
LoadEntity: () => import('./Iceberg'),
},
+ {
+ label: 'Hudi',
+ value: 'HUDI',
+ LoadEntity: () => import('./Hudi'),
+ },
{
label: 'Kafka',
value: 'KAFKA',
