This is an automated email from the ASF dual-hosted git repository.
leezng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 75c258294 [INLONG-6861][Dashboard] Support extract node of Apache Hudi
(#6873)
75c258294 is described below
commit 75c258294333f5d53538bb08a6a61c07c48425e8
Author: ZuoFengZhang <[email protected]>
AuthorDate: Mon Dec 19 09:10:04 2022 +0800
[INLONG-6861][Dashboard] Support extract node of Apache Hudi (#6873)
---
inlong-dashboard/src/locales/cn.json | 9 ++
inlong-dashboard/src/locales/en.json | 9 ++
.../src/metas/sources/defaults/Hudi.ts | 145 +++++++++++++++++++++
3 files changed, 163 insertions(+)
diff --git a/inlong-dashboard/src/locales/cn.json
b/inlong-dashboard/src/locales/cn.json
index 9faf4b2da..021ef47a3 100644
--- a/inlong-dashboard/src/locales/cn.json
+++ b/inlong-dashboard/src/locales/cn.json
@@ -92,6 +92,15 @@
"meta.Sources.MQTT.Username": "用户",
"meta.Sources.MQTT.Password": "密码",
"meta.Sources.MQTT.MqttVersion": "MQTT 版本",
+ "meta.Sources.Hudi.DbName": "库名",
+ "meta.Sources.Hudi.TableName": "表名",
+ "meta.Sources.Hudi.Warehouse": "仓库路径",
+ "meta.Sources.Hudi.ReadStreamingSkipCompaction": "跳过合并中的提交",
+ "meta.Sources.Hudi.ReadStartCommit": "起始的Commit",
+ "meta.Sources.Hudi.ReadStreamingSkipCompactionHelp": "流读时是否跳过 compaction 的
commits，跳过 compaction 有两个用途：1）避免 upsert 语义 下重复消费(compaction 的 instant
为重复数据，如果不跳过，有小概率会重复消费）; 2) changelog 模式下保证语义正确性。 0.11 开始，以上两个问题已经通过保留
compaction 的 instant time 修复",
+ "meta.Sources.Hudi.ReadStartCommitHelp": "指定
'yyyyMMddHHmmss'格式的起始commit（闭区间）",
+ "meta.Sources.Hudi.ExtListHelper": "hudi表的DDL属性需带前缀'ddl.'",
+ "meta.Sources.Hudi.ExtList": "高级属性",
"meta.Sinks.SinkName": "名称",
"meta.Sinks.SinkNameRule": "以英文字母开头，只能包含英文字母、数字、中划线、下划线",
"meta.Sinks.SinkType": "类型",
diff --git a/inlong-dashboard/src/locales/en.json
b/inlong-dashboard/src/locales/en.json
index 0d599a6de..0e5c22a1c 100644
--- a/inlong-dashboard/src/locales/en.json
+++ b/inlong-dashboard/src/locales/en.json
@@ -92,6 +92,15 @@
"meta.Sources.MQTT.Username": "Username",
"meta.Sources.MQTT.Password": "Password",
"meta.Sources.MQTT.MqttVersion": "MQTT Version",
+ "meta.Sources.Hudi.DbName": "DbName",
+ "meta.Sources.Hudi.TableName": "TableName",
+ "meta.Sources.Hudi.Warehouse": "Warehouse",
+ "meta.Sources.Hudi.ReadStreamingSkipCompaction": "SkipCompaction",
+ "meta.Sources.Hudi.ReadStartCommit": "StartCommit",
+ "meta.Sources.Hudi.ReadStreamingSkipCompactionHelp": "Whether to skip
compaction commits during stream reading, skipping compaction has two purposes:
1) Avoid repeated consumption under upsert semantics (the instant of compaction
is repeated data, if not skipped, there is a small probability of repeated
consumption); 2) changelog Semantic correctness is guaranteed in the mode.
Starting from 0.11, the above two problems have been fixed by retaining the
instant time of compaction",
+ "meta.Sources.Hudi.ReadStartCommitHelp": "Specify the starting commit in the
format of 'yyyyMMddHHmmss' (closed range)",
+ "meta.Sources.Hudi.ExtListHelper": "The DDL attribute of the hudi table
needs to be prefixed with 'ddl.'",
+ "meta.Sources.Hudi.ExtList": "AdvancedProperties",
"meta.Sinks.SinkName": "Name",
"meta.Sinks.SinkNameRule": "At the beginning of English letters, only
English letters, numbers, minus, and underscores",
"meta.Sinks.SinkType": "Type",
diff --git a/inlong-dashboard/src/metas/sources/defaults/Hudi.ts
b/inlong-dashboard/src/metas/sources/defaults/Hudi.ts
new file mode 100644
index 000000000..326c00034
--- /dev/null
+++ b/inlong-dashboard/src/metas/sources/defaults/Hudi.ts
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { DataWithBackend } from '@/metas/DataWithBackend';
+import { RenderRow } from '@/metas/RenderRow';
+import { RenderList } from '@/metas/RenderList';
+import { SourceInfo } from '../common/SourceInfo';
+import i18n from '@/i18n';
+import EditableTable from '@/components/EditableTable';
+
+const { I18n } = DataWithBackend;
+const { FieldDecorator } = RenderRow;
+const { ColumnDecorator } = RenderList;
+
+export default class HudiSource
+ extends SourceInfo
+ implements DataWithBackend, RenderRow, RenderList
+{
+ @FieldDecorator({
+ type: 'input',
+ rules: [{ required: true }],
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sources.Hudi.DbName')
+ dbName: string;
+
+ @FieldDecorator({
+ type: 'input',
+ rules: [{ required: true }],
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sources.Hudi.TableName')
+ tableName: string;
+
+ @FieldDecorator({
+ type: 'input',
+ rules: [{ required: true }],
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ placeholder: 'thrift://127.0.0.1:9083',
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('Catalog URI')
+ catalogUri: string;
+
+ @FieldDecorator({
+ type: 'input',
+ rules: [{ required: true }],
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ placeholder: 'hdfs://127.0.0.1:9000/user/hudi/warehouse',
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sources.Hudi.Warehouse')
+ warehouse: string;
+
+ @FieldDecorator({
+ type: 'radio',
+ rules: [{ required: true }],
+ initialValue: false,
+ props: values => ({
+ disabled: values?.status === 101,
+ options: [
+ {
+ label: i18n.t('basic.Yes'),
+ value: true,
+ },
+ {
+ label: i18n.t('basic.No'),
+ value: false,
+ },
+ ],
+ }),
+ tooltip: i18n.t('meta.Sources.Hudi.ReadStreamingSkipCompactionHelp'),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sources.Hudi.ReadStreamingSkipCompaction')
+ readStreamingSkipCompaction: boolean;
+
+ @FieldDecorator({
+ type: 'input',
+ rules: [{ required: true }],
+ props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ placeholder: '20221213211100',
+ }),
+ tooltip: i18n.t('meta.Sources.Hudi.ReadStartCommitHelp'),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sources.Hudi.ReadStartCommit')
+ readStartCommit: string;
+
+ @FieldDecorator({
+ type: EditableTable,
+ rules: [{ required: false }],
+ initialValue: [],
+ tooltip: i18n.t('meta.Sources.Hudi.ExtListHelper'),
+ props: values => ({
+ size: 'small',
+ columns: [
+ {
+ title: 'Key',
+ dataIndex: 'keyName',
+ props: {
+ disabled: [110, 130].includes(values?.status),
+ },
+ },
+ {
+ title: 'Value',
+ dataIndex: 'keyValue',
+ props: {
+ disabled: [110, 130].includes(values?.status),
+ },
+ },
+ ],
+ }),
+ })
+ @ColumnDecorator()
+ @I18n('meta.Sources.Hudi.ExtList')
+ extList: string;
+}
