This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a change to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
discard b2350e10d9 [INLONG-10360][Manager] Combine schedule state transition
with group operations (#10445)
omit 8a6a490fc3 [INLONG-10436][Manager] Move schedule configuration from
stream to group (#10437)
omit a75f0576f5 [INLONG-10396][Manager] Support build-in schedule base on
quartz (#10412)
omit 08c520d5e4 [INLONG-10395][Manager] Add interface for schedule client
and engine (#10397)
omit b68bf5c5e5 [INLONG-10247][Manager] Support schedule information
management for offline sync (#10254)
omit b5279c097b [INLONG-9862][Manager] Support submit flink job for offline
sync (#9865)
omit ed21473a97 [INLONG-9822][Manager] Support flink job runtime execution
mode configuration (#9823)
omit 7b0ba77d0c [INLONG-9813][Manager] Support offline data sync management
(#9814)
omit a23ad3234b [INLONG-9781][Manager] Add offline sync task type
definition (#9787)
add 4bd806e9fc [INLONG-10365][Audit] Optimizing TCP sticky packets may
lead to duplication of audit data (#10366)
add 26927f526f [INLONG-10330][Manager] Support field template management
(#10333)
add e9cbc3671f [INLONG-10370][Manager] Support configuration of kV data
format (#10371)
add 92a6758d6a [INLONG-10373][Manager] Manager client support template
operation (#10374)
add fe4ba9ce8f [INLONG-10375][Manager] Add field mapping strategy for CLS,
StarRocks and Elasticsearch (#10376)
add 6b4e028278 Add HDFS Audit items in the Audit SDK (#10380)
add 09a1f073ec [INLONG-10321][Audit] Audit supports the Audit Proxy
service discovery and management (#10386)
add 1ba62ecc0b [INLONG-10363][Manager] Support template multi tenant
management (#10372)
add 505f217bc5 [INLONG-10284][Manager][Sort] Upgrade flink default version
to 1.15 (#10310)
add 08740777c3 [INLONG-10382][Manager] Support obtaining node information
through unique keys (#10383)
add 1001f66fed [INLONG-10388][Manager] Provide an interface to obtain the
audit proxy address (#10389)
add a0f123b676 [INLONG-10384][Agent] Add functions to the Store interface
to extend new storage plugins (#10385)
add 6c94610372 [INLONG-10377][Dashboard] Add Source Data Field Template
(#10378)
add 3186d0a43b [INLONG-10391][Manager] Supports configuring zk clusters
and issuing zk addresses to agents (#10392)
add 6c48c78dd3 [INLONG-10399][Agent] Add global configurations updater
(#10400)
add dc33a2ac65 [INLONG-10405][Manager] Fix the problem of Iceberg field
type mapping error (#10406)
add 7f5beb96e9 [INLONG-10407][Manager] Increase the interval for
determining heartbeat timeout (#10408)
add 0dd43462bd [INLONG-10381][Sort] Kafka source connector report audit
attach input time (#10393)
add e3d2b8dffa [INLONG-10261][Manager] Support installing agents by SSH
key-based authentication (#10390)
add c7e86ed1ea [INLONG-10394][DashBoard] Agent page modifies the version
and sets the def… (#10403)
add 6da93a43d1 [INLONG-10410][Agent] Add ZK plugin to save offset info
(#10411)
add aa6ec09701 [INLONG-10415][Manager] Kafka source supports configuring
wraptype (#10416)
add 315692d629 [INLONG-10419][Dashboard] Automatically switch tenants when
opening a page with groupId (#10420)
add 5fa60262c7 [INLONG-10387][Audit] Audit SDK supports obtaining
Audit-Proxy capabilities through InLong Manager (#10398)
add 08d1e5dddd [INLONG-10417][Dashboard] Fixed the group mode display
error (#10418)
add 17be87606d [INLONG-10413][Manager] Support for configuring built-in
fields for mysql and kafka (#10414)
add a97585eabf [INLONG-10421][Dashboard] Move template management to
system management (#10422)
add 3fdadcbefd [INLONG-10279][Manager] Support compress unified
configuration (#10280)
add 7230ab677b [INLONG-10423][Manager] Modify unified configuration
related classes and interfaces (#10424)
add 89699deee5 [INLONG-10425][Manager] Fix the problem of the kafka sink
defines duplicate attributes (#10426)
add 0a106ec6c7 [INLONG-10401][Sort] Add audit time metadata for MySQL
connector and relocate debezium dependencies (#10404)
add b8608e9d40 [INLONG-10427][SDK] The Go SDK supports authentication for
Manager access (#10427) (#10429)
add c725b85b17 [INLONG-10430][Manager] Check update error of unified
configuration (#10431)
add 3b2fb20ec7 [INLONG-10432][Manager] Delete unused method getMetaConfig
(#10433)
add 5c16b6df40 [INLONG-10438][Manager] GetConfig does not throw an
exception when obtaining the zk address fails (#10439)
add 298a6b9c4a [INLONG-10402][Audit] Audit Service supports the hourly
Audit data one day ago (#10428)
add 6d3a131701 [INLONG-10441][DataProxy] Supports obtaining Audit-Proxy
through InLong Manager (#10442)
add cd475efdc3 [INLONG-10446][Agent] Adjusting audit SDK address settings
(#10447)
add 826aa39bbd [INLONG-10457][SDK] Fix auth spelling errors (#10458)
add ade73737e7 [INLONG-10452][Manager] Delete the method of querying audit
information through MySQL (#10454)
add b15f24b3ed [INLONG-10455][Manager] Fix Set KafkaDataNode ack fail
(#10456)
add 5c6aaeead7 [INLONG-10311][Sort] Implement TubeMQ Source report audit
information exactly once And fix consuming TubeMQ data twice (#10440)
add 21ebfe5849 [INLONG-10443][Agent] Put Rocksdb into the plugins module
(#10444)
add 35f153b36e [INLONG-10450][Sort] Add a new version SinkMetric class to
support report audit information exactly once (#10451)
add df192c4407 [INLONG-10461][Docker] Update the way to configure audice
service info (#10462)
new 051c9716c6 [INLONG-9781][Manager] Add offline sync task type
definition (#9787)
new 2674a5645c [INLONG-9813][Manager] Support offline data sync management
(#9814)
new fafcece220 [INLONG-9822][Manager] Support flink job runtime execution
mode configuration (#9823)
new 36afe37602 [INLONG-9862][Manager] Support submit flink job for offline
sync (#9865)
new 8bd3a20cb4 [INLONG-10247][Manager] Support schedule information
management for offline sync (#10254)
new 4d5c873efb [INLONG-10395][Manager] Add interface for schedule client
and engine (#10397)
new 0eb6fa750d [INLONG-10396][Manager] Support build-in schedule base on
quartz (#10412)
new c2f14111f7 [INLONG-10436][Manager] Move schedule configuration from
stream to group (#10437)
new 2a08660a07 [INLONG-10360][Manager] Combine schedule state transition
with group operations (#10445)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b2350e10d9)
\
N -- N -- N refs/heads/dev-offline-sync (2a08660a07)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docker/docker-compose/docker-compose.yml | 11 +-
.../inlong/agent/constant/AgentConstants.java | 9 +-
.../inlong/agent/constant/CommonConstants.java | 6 +-
.../inlong/agent/constant/FetcherConstants.java | 5 +-
.../inlong/agent/metrics/audit/AuditUtils.java | 23 +-
.../org/apache/inlong/agent/plugin/file/Task.java | 4 +-
.../InstanceDb.java => store/InstanceStore.java} | 45 +-
.../inlong/agent/{db => store}/KeyValueEntity.java | 2 +-
.../inlong/agent/{db => store}/LocalProfile.java | 2 +-
.../{db/OffsetDb.java => store/OffsetStore.java} | 46 +-
.../inlong/agent/{db => store}/StateSearchKey.java | 2 +-
.../inlong/agent/{db/Db.java => store/Store.java} | 12 +-
.../TaskProfileDb.java => store/TaskStore.java} | 36 +-
.../org/apache/inlong/agent/core/AgentManager.java | 15 +-
.../agent/core/instance/InstanceManager.java | 132 ++--
.../inlong/agent/core/task/OffsetManager.java | 53 +-
.../apache/inlong/agent/core/task/TaskManager.java | 219 +++----
.../agent/core/{task => }/TestMemoryManager.java | 3 +-
.../agent-installer/conf/installer.properties | 4 +-
.../org/apache/inlong/agent/installer/Manager.java | 2 +-
.../inlong/agent/installer/ModuleManager.java | 3 +-
.../installer/conf/InstallerConfiguration.java | 2 +-
.../agent/plugin/fetcher/ManagerFetcher.java | 83 ++-
.../inlong/agent/plugin/sources/LogFileSource.java | 2 +-
.../agent/plugin/sources/file/AbstractSource.java | 4 +-
.../agent/plugin/store/RocksDBStoreImpl.java} | 31 +-
.../inlong/agent/plugin/store/ZooKeeperImpl.java | 185 ++++++
.../inlong/agent/plugin/task/AbstractTask.java | 10 +-
.../apache/inlong/agent/plugin/task/CronTask.java | 4 +-
.../inlong/agent/plugin/utils/RocksDBUtils.java | 16 +-
.../inlong/agent/plugin/AgentBaseTestsHelper.java | 8 +-
.../agent/plugin}/instance/MockInstance.java | 5 +-
.../plugin}/instance/TestInstanceManager.java | 33 +-
.../inlong/agent/plugin/sinks/KafkaSinkTest.java | 3 +-
.../inlong/agent/plugin/sinks/PulsarSinkTest.java | 3 +-
.../sinks/filecollect/TestSenderManager.java | 3 +-
.../agent/plugin/sources/TestLogFileSource.java | 44 +-
.../agent/plugin/store/TestRocksDBStoreImpl.java} | 34 +-
.../inlong/agent/plugin/store/TestRocksDbKey.java | 69 ++
.../inlong/agent/plugin/store/TestStoreKey.java | 76 +++
.../apache/inlong/agent/plugin}/task/MockTask.java | 7 +-
.../inlong/agent/plugin/task/TestLogFileTask.java | 7 +-
.../inlong/agent/plugin}/task/TestTaskManager.java | 17 +-
inlong-agent/bin/agent-config.sh | 3 +-
inlong-agent/conf/agent.properties | 10 +-
inlong-audit/audit-common/pom.xml | 4 +
.../inlong/audit/entity/AuditComponent.java} | 41 +-
.../org/apache/inlong/audit/entity/AuditProxy.java | 33 +-
.../apache/inlong/audit/entity/CommonResponse.java | 91 +++
.../apache/inlong/audit/file/ConfigManager.java | 8 +
.../org/apache/inlong/audit/utils/HttpUtils.java | 106 +++
.../apache/inlong/audit/heartbeat/Heartbeat.java | 135 ++++
.../org/apache/inlong/audit/node/Application.java | 5 +
inlong-audit/audit-sdk/README.md | 48 +-
.../java/org/apache/inlong/audit/AuditIdEnum.java | 16 +-
.../org/apache/inlong/audit/AuditReporterImpl.java | 105 ++-
.../SenderResult.java => entity/AuditMetric.java} | 49 +-
.../org/apache/inlong/audit/entity/AuditType.java | 7 +-
.../org/apache/inlong/audit/send/ProxyManager.java | 151 +++++
.../apache/inlong/audit/send/SenderChannel.java | 215 -------
.../org/apache/inlong/audit/send/SenderGroup.java | 270 --------
.../apache/inlong/audit/send/SenderHandler.java | 110 ----
.../apache/inlong/audit/send/SenderManager.java | 360 +++++------
.../org/apache/inlong/audit/util/AuditConfig.java | 36 +-
.../inlong/audit/util/AuditManagerUtils.java | 5 +
.../java/org/apache/inlong/audit/util/Decoder.java | 57 --
.../apache/inlong/audit/util/EventLoopUtil.java | 118 ----
.../apache/inlong/audit/util/RequestIdUtils.java | 25 +-
...enderManagerTest.java => ProxyManagerTest.java} | 42 +-
.../apache/inlong/audit/send/SenderGroupTest.java | 53 --
.../inlong/audit/send/SenderManagerTest.java | 15 +-
.../inlong/audit/util/RequestIdUtilsTest.java | 39 +-
.../apache/inlong/audit/cache/AbstractCache.java | 81 ++-
.../apache/inlong/audit/cache/AuditProxyCache.java | 234 +++++++
.../inlong/audit/config/ConfigConstants.java | 8 +
.../inlong/audit/config/OpenApiConstants.java | 11 +-
.../apache/inlong/audit/config/SqlConstants.java | 69 ++
.../org/apache/inlong/audit/entities/ApiType.java | 2 +-
.../entities/{ApiType.java => CacheKeyEntity.java} | 15 +-
.../entities/{ApiType.java => Heartbeat.java} | 15 +-
.../inlong/audit/heartbeat/ProxyHeartbeat.java | 96 +++
.../apache/inlong/audit/service/ApiService.java | 57 +-
.../org/apache/inlong/audit/sink/JdbcSink.java | 68 +-
inlong-audit/sql/apache_inlong_audit_mysql.sql | 27 +
.../apache/inlong/common/constant/SinkType.java | 2 +
.../apache/inlong/common/enums/DataTypeEnum.java | 1 +
.../AgentConfigInfo.java} | 31 +-
.../AgentConfigRequest.java} | 22 +-
.../inlong/common/pojo/sort/SortClusterConfig.java | 50 +-
.../common/pojo/sort/SortConfigResponse.java | 2 +-
.../sort/dataflow/dataType/DataTypeConfig.java | 2 +-
.../deserialization/DeserializationConfig.java | 2 +-
.../common/pojo/sort/dataflow/sink/SinkConfig.java | 2 +-
.../common/pojo/sort/mq/MqClusterConfig.java | 2 +-
.../inlong/common/pojo/sort/node/NodeConfig.java | 2 +-
.../org/apache/inlong/common/util/StringUtil.java | 9 +
inlong-dashboard/src/configs/menus/conf.tsx | 4 +
inlong-dashboard/src/configs/routes/conf.ts | 4 +
inlong-dashboard/src/core/utils/pattern.ts | 1 +
inlong-dashboard/src/i18n.ts | 4 +
.../plugins/groups/common/GroupDataTemplateInfo.ts | 171 +++++
.../plugins/streams/common/StreamDefaultInfo.ts | 30 +
.../src/ui/components/Layout/Tenant/index.tsx | 2 +-
inlong-dashboard/src/ui/locales/cn.json | 14 +-
inlong-dashboard/src/ui/locales/en.json | 12 +-
.../src/ui/pages/AgentModule/CreateModal.tsx | 25 +-
.../src/ui/pages/Clusters/NodeEditModal.tsx | 95 ++-
.../src/ui/pages/GroupDataTemplate/CreateModal.tsx | 112 ++++
.../src/ui/pages/GroupDataTemplate/index.tsx | 195 ++++++
.../GroupDetail/DataStream/StreamItemModal.tsx | 21 +
.../src/ui/pages/GroupDetail/index.tsx | 15 +
.../src/ui/pages/Process/Applies/config.tsx | 10 +-
.../src/ui/pages/Process/Approvals/config.tsx | 6 +-
inlong-dataproxy/conf/common.properties | 2 +
.../dataproxy/config/CommonConfigHolder.java | 23 +
.../inlong/dataproxy/metrics/audit/AuditUtils.java | 12 +-
.../src/test/resources/common.properties | 2 +
.../client/api/inner/client/TemplateClient.java | 123 ++++
.../manager/client/api/service/TemplateApi.java | 54 ++
.../manager/common/consts/InlongConstants.java | 2 +
.../inlong/manager/common/consts/StreamType.java | 2 +-
.../inlong/manager/common/enums/ClusterType.java | 2 +
.../inlong/manager/common/enums/ErrorCodeEnum.java | 8 +-
.../manager/common/enums/OperationTarget.java | 2 +
...tQuerySource.java => TemplateVisibleRange.java} | 12 +-
.../strategy/ClsFieldTypeStrategy.java} | 43 +-
.../strategy/ElasticsearchFieldTypeStrategy.java | 26 +-
.../strategy/StarRocksFieldTypeStrategy.java | 26 +-
.../src/main/resources/cls-field-type-mapping.yaml | 231 +++++++
.../src/main/resources/es-field-type-mapping.yaml | 231 +++++++
.../main/resources/iceberg-field-type-mapping.yaml | 2 +-
.../inlong/manager/dao/entity/TemplateEntity.java | 36 +-
.../manager/dao/entity/TemplateFieldEntity.java | 33 +-
.../manager/dao/entity/TenantTemplateEntity.java | 34 +-
.../manager/dao/mapper/AuditEntityMapper.java | 69 --
.../manager/dao/mapper/TemplateEntityMapper.java} | 35 +-
.../dao/mapper/TemplateFieldEntityMapper.java | 55 +-
.../dao/mapper/TenantTemplateEntityMapper.java | 55 +-
.../main/resources/mappers/AuditEntityMapper.xml | 105 ---
.../resources/mappers/TemplateEntityMapper.xml | 109 ++++
.../mappers/TemplateFieldEntityMapper.xml | 122 ++++
.../mappers/TenantTemplateEntityMapper.xml | 112 ++++
inlong-manager/manager-docker/Dockerfile | 17 +-
inlong-manager/manager-docker/manager-docker.sh | 1 +
inlong-manager/manager-docker/pom.xml | 20 +-
inlong-manager/manager-plugins/base/pom.xml | 6 +-
.../main/resources/flink-sort-plugin.properties | 2 +-
.../manager/pojo/audit/AuditProxyResponse.java | 28 +-
.../manager/pojo/cluster/zk/AgentZkClusterDTO.java | 63 ++
.../zk/AgentZkClusterInfo.java} | 36 +-
.../zk/AgentZkClusterRequest.java} | 32 +-
.../pojo/node/kafka/KafkaDataNodeRequest.java | 2 +-
.../inlong/manager/pojo/sink/cls/ClsSink.java | 9 +
.../pojo/sort/node/base/ExtractNodeProvider.java | 20 +-
.../pojo/sort/node/provider/DorisProvider.java | 1 -
.../pojo/sort/node/provider/KafkaProvider.java | 25 +
.../sort/node/provider/MySQLBinlogProvider.java | 24 +
.../pojo/sort/node/provider/PulsarProvider.java | 2 +
.../pojo/sort/node/provider/TubeMqProvider.java | 2 +
.../manager/pojo/source/kafka/KafkaSource.java | 6 +-
.../manager/pojo/source/kafka/KafkaSourceDTO.java | 11 +-
.../pojo/source/kafka/KafkaSourceRequest.java | 12 +-
.../manager/pojo/source/pulsar/PulsarSource.java | 3 +
.../pojo/source/pulsar/PulsarSourceDTO.java | 8 +-
.../pojo/source/pulsar/PulsarSourceRequest.java | 9 +-
.../manager/pojo/source/tubemq/TubeMQSource.java | 9 +
.../pojo/source/tubemq/TubeMQSourceDTO.java | 12 +-
.../pojo/source/tubemq/TubeMQSourceRequest.java | 9 +
.../inlong/manager/pojo/stream/TemplateField.java | 117 ++++
.../inlong/manager/pojo/stream/TemplateInfo.java | 59 ++
.../manager/pojo/stream/TemplatePageRequest.java | 62 ++
.../manager/pojo/stream/TemplateRequest.java | 59 ++
.../manager/pojo/stream/TenantTemplateInfo.java | 63 ++
.../pojo/stream/TenantTemplatePageRequest.java | 27 +-
.../manager/pojo/stream/TenantTemplateRequest.java | 53 ++
.../service/cluster/AgentZkClusterOperator.java | 94 +++
.../service/cluster/InlongClusterService.java | 26 +-
.../service/cluster/InlongClusterServiceImpl.java | 57 +-
.../manager/service/cmd/CommandExecutor.java | 2 +
.../manager/service/cmd/CommandExecutorImpl.java | 24 +-
.../service/cmd/shell/ShellExecutorImpl.java | 13 +-
.../inlong/manager/service/core/AgentService.java | 10 +
.../inlong/manager/service/core/AuditService.java | 8 +
.../service/core/impl/AgentServiceImpl.java | 33 +
.../service/core/impl/AuditServiceImpl.java | 184 +-----
.../manager/service/core/impl/SortServiceImpl.java | 24 +-
.../service/heartbeat/HeartbeatManager.java | 2 +-
.../service/node/cls/ClsDataNodeOperator.java | 2 +
.../node/es/ElasticsearchDataNodeOperator.java | 2 +
.../service/node/kafka/KafkaDataNodeOperator.java | 2 +
.../node/pulsar/PulsarDataNodeOperator.java | 2 +
.../repository/DataProxyConfigRepository.java | 2 -
.../repository/DataProxyConfigRepositoryV2.java | 714 ---------------------
.../resource/sort/DefaultSortConfigOperator.java | 37 +-
.../manager/service/sink/AbstractSinkOperator.java | 6 +-
.../manager/service/sink/StreamSinkOperator.java | 7 +-
.../manager/service/sink/cls/ClsSinkOperator.java | 7 +-
.../service/sink/es/ElasticsearchSinkOperator.java | 10 +-
.../service/sink/kafka/KafkaSinkOperator.java | 7 +-
.../service/sink/pulsar/PulsarSinkOperator.java | 7 +-
.../service/stream/InlongStreamServiceImpl.java | 1 -
.../manager/service/stream/TemplateService.java | 83 +++
.../service/stream/TemplateServiceImpl.java | 334 ++++++++++
.../resources/application-unit-test.properties | 12 -
.../main/resources/h2/apache_inlong_manager.sql | 59 ++
.../manager-web/sql/apache_inlong_manager.sql | 61 ++
inlong-manager/manager-web/sql/changes-1.13.0.sql | 64 +-
.../manager/web/controller/AuditController.java | 10 +
.../manager/web/controller/DataNodeController.java | 11 +
.../web/controller/InlongClusterController.java | 6 +
.../manager/web/controller/TemplateController.java | 106 +++
.../web/controller/openapi/AgentController.java | 8 +
.../web/controller/openapi/AuditController.java | 11 +
.../controller/openapi/DataProxyController.java | 11 +-
.../web/controller/openapi/SortController.java | 2 +-
.../src/main/resources/application-dev.properties | 2 -
.../src/main/resources/application-prod.properties | 2 -
.../src/main/resources/application-test.properties | 2 -
.../manager-web/src/main/resources/exec_cmd.exp | 8 +-
.../src/main/resources/ssh_key_cmd.exp} | 34 +-
.../dataproxy-sdk-golang/dataproxy/client.go | 2 +-
.../dataproxy-sdk-golang/dataproxy/discoverer.go | 36 +-
.../dataproxy-sdk-golang/dataproxy/options.go | 1 +
.../dataproxy/options_basic.go | 10 +
.../loader/v2/ManagerSortClusterConfigLoader.java | 5 +-
inlong-sort/pom.xml | 6 +-
.../protocol/node/extract/KafkaExtractNode.java | 11 +-
.../protocol/node/extract/MySqlExtractNode.java | 4 +-
.../node/extract/KafkaExtractNodeTest.java | 1 +
.../node/extract/MySqlExtractNodeTest.java | 1 +
inlong-sort/sort-core/pom.xml | 6 +-
inlong-sort/sort-dist/pom.xml | 6 +-
inlong-sort/sort-flink/base/pom.xml | 6 +-
.../inlong/sort/base/metric/MetricsCollector.java | 11 +-
.../inlong/sort/base/metric/SinkExactlyMetric.java | 333 ++++++++++
...rceMetricData.java => SourceExactlyMetric.java} | 152 ++---
.../inlong/sort/base/metric/SourceMetricData.java | 7 +-
.../sort/base/metric/SourceMetricsReporter.java | 11 +-
.../sort-flink-v1.15/sort-connectors/kafka/pom.xml | 6 +-
.../table/DynamicKafkaDeserializationSchema.java | 27 +-
.../sort/kafka/table/KafkaDynamicSource.java | 13 +
.../sort-connectors/mysql-cdc/pom.xml | 4 +
.../apache/inlong/sort/mysql/MySqlTableSource.java | 2 +-
.../mysql/RowDataDebeziumDeserializeSchema.java | 19 +-
.../sort/mysql/table/MySqlReadableMetadata.java | 128 ++++
.../inlong/sort/tubemq/FlinkTubeMQConsumer.java | 53 +-
.../table/DynamicTubeMQDeserializationSchema.java | 6 +-
.../DynamicTubeMQTableDeserializationSchema.java | 27 +-
.../format-row/format-json-v1.13/pom.xml | 2 +-
.../format-row/format-json-v1.15/pom.xml | 2 +-
.../format-row/format-json-v1.18/pom.xml | 2 +-
.../CanalJsonEnhancedDeserializationSchema.java | 9 +
.../CanalJsonEnhancedSerializationSchema.java | 5 +
.../canal/CanalJsonEnhancedSerDeSchemaTest.java | 1 +
inlong-sort/sort-formats/format-row/pom.xml | 10 +-
inlong-sort/sort-formats/pom.xml | 6 +-
licenses/inlong-sort-connectors/LICENSE | 1 +
pom.xml | 6 +-
258 files changed, 6914 insertions(+), 3478 deletions(-)
rename
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/{db/InstanceDb.java
=> store/InstanceStore.java} (70%)
rename inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/{db =>
store}/KeyValueEntity.java (98%)
rename inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/{db =>
store}/LocalProfile.java (98%)
rename
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/{db/OffsetDb.java
=> store/OffsetStore.java} (69%)
rename inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/{db =>
store}/StateSearchKey.java (96%)
copy
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/{db/Db.java =>
store/Store.java} (86%)
rename
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/{db/TaskProfileDb.java
=> store/TaskStore.java} (71%)
rename
inlong-agent/agent-core/src/test/java/org/apache/inlong/agent/core/{task =>
}/TestMemoryManager.java (97%)
rename
inlong-agent/{agent-common/src/main/java/org/apache/inlong/agent/db/RocksDbImp.java
=>
agent-plugins/src/main/java/org/apache/inlong/agent/plugin/store/RocksDBStoreImpl.java}
(91%)
create mode 100755
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/store/ZooKeeperImpl.java
rename inlong-agent/{agent-core/src/test/java/org/apache/inlong/agent/core =>
agent-plugins/src/test/java/org/apache/inlong/agent/plugin}/instance/MockInstance.java
(92%)
rename inlong-agent/{agent-core/src/test/java/org/apache/inlong/agent/core =>
agent-plugins/src/test/java/org/apache/inlong/agent/plugin}/instance/TestInstanceManager.java
(83%)
rename
inlong-agent/{agent-common/src/test/java/org/apache/inlong/agent/db/TestRocksDbImp.java
=>
agent-plugins/src/test/java/org/apache/inlong/agent/plugin/store/TestRocksDBStoreImpl.java}
(70%)
create mode 100644
inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/store/TestRocksDbKey.java
create mode 100644
inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/store/TestStoreKey.java
rename inlong-agent/{agent-core/src/test/java/org/apache/inlong/agent/core =>
agent-plugins/src/test/java/org/apache/inlong/agent/plugin}/task/MockTask.java
(88%)
rename inlong-agent/{agent-core/src/test/java/org/apache/inlong/agent/core =>
agent-plugins/src/test/java/org/apache/inlong/agent/plugin}/task/TestTaskManager.java
(91%)
copy
inlong-audit/{audit-sdk/src/main/java/org/apache/inlong/audit/entity/AuditType.java
=>
audit-common/src/main/java/org/apache/inlong/audit/entity/AuditComponent.java}
(61%)
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/SortConfigResponse.java
=>
inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entity/AuditProxy.java
(65%)
create mode 100644
inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entity/CommonResponse.java
create mode 100644
inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/HttpUtils.java
create mode 100644
inlong-audit/audit-proxy/src/main/java/org/apache/inlong/audit/heartbeat/Heartbeat.java
rename
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/{util/SenderResult.java
=> entity/AuditMetric.java} (54%)
create mode 100644
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/send/ProxyManager.java
delete mode 100644
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/send/SenderChannel.java
delete mode 100644
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/send/SenderGroup.java
delete mode 100644
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/send/SenderHandler.java
delete mode 100644
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/Decoder.java
delete mode 100644
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/EventLoopUtil.java
copy
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/AuditQuerySource.java
=>
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/RequestIdUtils.java
(61%)
copy
inlong-audit/audit-sdk/src/test/java/org/apache/inlong/audit/send/{SenderManagerTest.java
=> ProxyManagerTest.java} (52%)
delete mode 100644
inlong-audit/audit-sdk/src/test/java/org/apache/inlong/audit/send/SenderGroupTest.java
copy
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/OperationTarget.java
=>
inlong-audit/audit-sdk/src/test/java/org/apache/inlong/audit/util/RequestIdUtilsTest.java
(63%)
create mode 100644
inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/cache/AuditProxyCache.java
copy
inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/{ApiType.java
=> CacheKeyEntity.java} (80%)
copy
inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/{ApiType.java
=> Heartbeat.java} (82%)
create mode 100644
inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/heartbeat/ProxyHeartbeat.java
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/{sort/SortConfigResponse.java
=> agent/AgentConfigInfo.java} (71%)
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/{sort/SortConfigResponse.java
=> agent/AgentConfigRequest.java} (72%)
create mode 100644
inlong-dashboard/src/plugins/groups/common/GroupDataTemplateInfo.ts
create mode 100644
inlong-dashboard/src/ui/pages/GroupDataTemplate/CreateModal.tsx
create mode 100644 inlong-dashboard/src/ui/pages/GroupDataTemplate/index.tsx
create mode 100644
inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/TemplateClient.java
create mode 100644
inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/service/TemplateApi.java
rename
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/{AuditQuerySource.java
=> TemplateVisibleRange.java} (88%)
copy
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/{enums/OperationTarget.java
=> fieldtype/strategy/ClsFieldTypeStrategy.java} (58%)
copy
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/send/ClientPipelineFactory.java
=>
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/ElasticsearchFieldTypeStrategy.java
(57%)
rename
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/send/ClientPipelineFactory.java
=>
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/StarRocksFieldTypeStrategy.java
(57%)
create mode 100644
inlong-manager/manager-common/src/main/resources/cls-field-type-mapping.yaml
create mode 100644
inlong-manager/manager-common/src/main/resources/es-field-type-mapping.yaml
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/SortConfigResponse.java
=>
inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/TemplateEntity.java
(60%)
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/dataType/DataTypeConfig.java
=>
inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/TemplateFieldEntity.java
(55%)
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/SortConfigResponse.java
=>
inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/TenantTemplateEntity.java
(63%)
delete mode 100644
inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/AuditEntityMapper.java
copy
inlong-manager/{manager-common/src/main/java/org/apache/inlong/manager/common/enums/OperationTarget.java
=>
manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/TemplateEntityMapper.java}
(59%)
copy
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/Db.java =>
inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/TemplateFieldEntityMapper.java
(53%)
rename
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/Db.java =>
inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/TenantTemplateEntityMapper.java
(54%)
delete mode 100644
inlong-manager/manager-dao/src/main/resources/mappers/AuditEntityMapper.xml
create mode 100644
inlong-manager/manager-dao/src/main/resources/mappers/TemplateEntityMapper.xml
create mode 100644
inlong-manager/manager-dao/src/main/resources/mappers/TemplateFieldEntityMapper.xml
create mode 100644
inlong-manager/manager-dao/src/main/resources/mappers/TenantTemplateEntityMapper.xml
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/SortConfigResponse.java
=>
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/audit/AuditProxyResponse.java
(69%)
create mode 100644
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/cluster/zk/AgentZkClusterDTO.java
copy
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/{node/kafka/KafkaDataNodeRequest.java
=> cluster/zk/AgentZkClusterInfo.java} (57%)
copy
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/{node/kafka/KafkaDataNodeRequest.java
=> cluster/zk/AgentZkClusterRequest.java} (57%)
create mode 100644
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/TemplateField.java
create mode 100644
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/TemplateInfo.java
create mode 100644
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/TemplatePageRequest.java
create mode 100644
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/TemplateRequest.java
create mode 100644
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/TenantTemplateInfo.java
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/SortConfigResponse.java
=>
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/TenantTemplatePageRequest.java
(59%)
create mode 100644
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/TenantTemplateRequest.java
create mode 100644
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/cluster/AgentZkClusterOperator.java
delete mode 100644
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/repository/DataProxyConfigRepositoryV2.java
create mode 100644
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/stream/TemplateService.java
create mode 100644
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/stream/TemplateServiceImpl.java
create mode 100644
inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/TemplateController.java
copy
inlong-manager/{manager-plugins/base/src/main/resources/flink-sort-plugin.properties
=> manager-web/src/main/resources/ssh_key_cmd.exp} (62%)
create mode 100644
inlong-sort/sort-flink/base/src/main/java/org/apache/inlong/sort/base/metric/SinkExactlyMetric.java
copy
inlong-sort/sort-flink/base/src/main/java/org/apache/inlong/sort/base/metric/{SourceMetricData.java
=> SourceExactlyMetric.java} (66%)
copy
inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/ApiType.java
=>
inlong-sort/sort-flink/base/src/main/java/org/apache/inlong/sort/base/metric/SourceMetricsReporter.java
(84%)
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/mysql/table/MySqlReadableMetadata.java
