This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a change to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
discard a793963543 [INLONG-10247][Manager] Support schedule information
management for offline sync (#10254)
omit 65d3a928ea [INLONG-9862][Manager] Support submit flink job for offline
sync (#9865)
omit ff3c351401 [INLONG-9822][Manager] Support flink job runtime execution
mode configuration (#9823)
omit dfb22f4d8e [INLONG-9813][Manager] Support offline data sync management
(#9814)
omit 1e0fddc0a8 [INLONG-9781][Manager] Add offline sync task type
definition (#9787)
add 03c2160411 [INLONG-10225][Audit] Audit SDK provides the ability to
automatically assign and manage Audit ID (#10232)
add 40548ae01b [INLONG-10229][Sort] EsSink support unified configuration
(#10237)
add 5a331f1c86 [INLONG-10230][Sort] KafkaSink support unified
configuration (#10235)
add 97e12593e2 [INLONG-10233][Manager] Report heartbeat removal port
restriction (#10234)
add e4e24def90 [INLONG-10228][Sort] PulsarSink support unified
configuration (#10236)
add 6645261800 [INLONG-10245][Manager] Support setting audit version for
file collection (#10246)
add 2acf82d174 [INLONG-10238][Sort] MySQL connector support audit ID
(#10239)
add 0acdfe9519 [INLONG-10195][Sort] Hudi connector support audit ID
(#10231)
add 23728055c3 [INLONG-10242][Audit] Audit SDK compatible with InLong
Manager to manage audit items (#10248)
add 59ea439321 [INLONG-10249][Manager] Fix the problem of duplicate data
appears during data preview (#10250)
add 85fc8b02e6 [INLONG-9613][TubeMQ] Adjust FATAL type error return
content, without carrying class name (#10251)
add 6eeec6ee61 [INLONG-10252][Docker] Fix the bug can not get the real
audit address (#10253)
add d890a1933f [INLONG-10260][Sort] Correct wrong create table statement
(#10262)
add 2c890c29bb [INLONG-10263][Audit] Solve the conflict between the
jdbc.url parameter of Audit Store and the container environment variable
(#10264)
add d18232b54a [INLONG-10241][Sort] TLog format requires the ability to
parse and determine if the first segment is present (#10243)
add b5261a3927 [INLONG-10224][Sort] Unified configuration check utils
(#10255)
add 94a99ac999 [INLONG-10266][Manager] Fix the problem of password is
overwritten when adding field information (#10267)
add 0113074789 [INLONG-10272][Sort] Unified configuration check utils
support check latest config (#10273)
add b22a3bdca8 [INLONG-10256][DashBoard] Modify the data source IP item of
the file data source form in the data access module (#10258)
add ca4bc8dce9 [INLONG-10268][Agent] Get the data version from the
auditVersion field (#10269)
add ca3fd42542 [INLONG-10270][Manager] Data source tasks allow for
multiple IPs (#10271)
add 9f829fac3b [INLONG-10274][Audit] The OpenAPI of Audit Service returns
the average transmission time (#10275)
add 879bf36017 [INLONG-10265][Manager] Fix the problem of correct wrong
starrocks create table statement (#10276)
add 4f9c4098eb [INLONG-10281][Agent] Real time file collection takes the
current time as the data time (#10282)
add e0a564b0b2 [INLONG-10277][Manager] Support calling API to refresh
cluster config (#10278)
add af0cc8255a [INLONG-10277][Manager] Fix the problem of migration failed
when migrating multiple groups to the same tenant (#10285)
add 863190ea29 [INLONG-10290][Manager] Prohibit groups that have not been
successfully configured from obtaining dataproxy addresses (#10293)
add c07316253e [INLONG-10292][SDK] Fix panic in connpool.UpdateEndpoints()
of Golang SDK (#10295)
add d43e1e84ec [INLONG-10291][SDK] Fix incorrect initializing of gnet in
Golang SDK (#10294)
add 4555ebfe1c [INLONG-10297][Sort] Fix mysql connector cannot submit to
flink cluster (#10301)
add e5ad544716 [INLONG-10306][Audit] Compatible with scenarios where the
Audit Tag is empty (#10308)
add 12efb53dae [INLONG-10302][Agent] Add an interface for limiting the
number of instances obtained (#10303)
add b0a70bda57 [INLONG-10300][Manager] Allow unsubmitted groups to modify
mq type (#10304)
add 229174675b [INLONG-10298][Agent] Delete useless code (#10299)
add 9d94ef72df [INLONG-10305][Manager] Delete k8s related parameters in
file collection (#10307)
add 83ded2149d [INLONG-10313][DataProxy] Replace audit ID macro with audit
API (#10315)
add c6892951ed [INLONG-10319][Agent] Get audit Id from audit sdk (#10322)
add 8f6ce51cd7 [INLONG-10314][DashBoard] Add an operation time to the
operation log table (#10316)
add ca0983fceb [INLONG-10318][Agent] Add PostgreSQL data source for Agent
(#10320)
add 83fc0711b8 [INLONG-10324][Manager] Fix the problem of
useExtendedFields in Stream incorrectly overwritten (#10325)
add 8171257a62 [INLONG-10332][Sort] Relocate the google.protobuf in
Iceberg Connector (#10334)
add 3329500de7 [INLONG-10223][Dashboard] Agent cluster node supports
configuring ssh information (#10244)
add 160253ca8b [INLONG-10335][Manager] Use audit sdk to obtain audit ID
(#10336)
add 8f3b920329 [INLONG-10345][Manager] Fix the problem of error in
obtaining audit id value (#10346)
add ba62f4037b [INLONG-10331][CVE] Unify and upgrade aircompressor version
and update LICENSE (#10337)
add 643c790e02 [INLONG-10326][Manager] Support obtaining tenant
information based on groupId (#10327)
add eadda7cd3d [INLONG-10347][Dashboard] Add Agent IP field to PostgreSQL
data source (#10348)
add ed2721f23c [INLONG-10351][Manager] Fix the problem of no audit name
returned when obtaining audit information (#10352)
add 36a2032acc [INLONG-10339][Sort] Fix PostgreSQL AuditOperator not
serialized (#10344)
add 24beb99161 [INLONG-10340][Sort] Fix MongoDB AuditOperator not
serialized (#10343)
add 9e2bc9d74f [INLONG-10338][Sort] Fix Sqlserver AuditOperator not
serialized (#10341)
add aef0b9dbee [INLONG-10328][Manager] Support automatic synchronization
of stream fields to sink (#10329)
add f9cb48e905 [INLONG-10350][Dashboard] Fix audit item search failure
(#10361)
add cdab4cd5e8 [INLONG-10353][Manager] Refactor code for building and
submitting flink job (#10354)
add 497f0966bd [INLONG-10362][Manager] Simplify code for InLong group
management (#10364)
add ce49ad9bc8 [INLONG-10368][Manager] Data preview supports obtaining
message attribute information (#10369)
add c244ec7561 [INLONG-10323][Sort] Support KV deserialization format in
sort module (#10349)
new a23ad3234b [INLONG-9781][Manager] Add offline sync task type
definition (#9787)
new 7b0ba77d0c [INLONG-9813][Manager] Support offline data sync management
(#9814)
new ed21473a97 [INLONG-9822][Manager] Support flink job runtime execution
mode configuration (#9823)
new b5279c097b [INLONG-9862][Manager] Support submit flink job for offline
sync (#9865)
new b68bf5c5e5 [INLONG-10247][Manager] Support schedule information
management for offline sync (#10254)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (a793963543)
\
N -- N -- N refs/heads/dev-offline-sync (b68bf5c5e5)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docker/docker-compose/docker-compose.yml | 1 +
.../apache/inlong/agent/conf/InstanceProfile.java | 4 +
.../inlong/agent/constant/TaskConstants.java | 14 +
.../java/org/apache/inlong/agent/db/CommandDb.java | 61 ---
.../main/java/org/apache/inlong/agent/db/Db.java | 75 +--
.../org/apache/inlong/agent/db/RocksDbImp.java | 104 +---
.../agent/message/file/ProxyMessageCache.java | 9 +-
.../inlong/agent/metrics/audit/AuditUtils.java | 49 +-
.../apache/inlong/agent/pojo/PostgreSQLTask.java | 9 +-
.../apache/inlong/agent/pojo/TaskProfileDto.java | 24 +-
.../org/apache/inlong/agent/db/TestRocksDbImp.java | 20 +-
.../inlong/agent/metrics/TestAuditUtils.java | 139 +++++
.../agent/core/instance/InstanceManager.java | 3 +-
.../inlong/agent/core/task/OffsetManager.java | 7 +-
.../apache/inlong/agent/core/task/TaskManager.java | 4 +
.../agent/plugin/instance/CommonInstance.java | 4 +-
.../agent/plugin/instance/PostgreSQLInstance.java | 16 +-
.../plugin/sinks/filecollect/SenderManager.java | 3 +-
.../inlong/agent/plugin/sources/LogFileSource.java | 4 +-
.../agent/plugin/sources/PostgreSQLSource.java | 160 ++++--
.../agent/plugin/sources/file/AbstractSource.java | 6 +-
.../inlong/agent/plugin/task/AbstractTask.java | 10 +-
.../apache/inlong/agent/plugin/task/KafkaTask.java | 5 +
.../inlong/agent/plugin/task/MongoDBTask.java | 5 +
.../task/{PulsarTask.java => PostgreSQLTask.java} | 69 ++-
.../inlong/agent/plugin/task/PulsarTask.java | 5 +
.../inlong/agent/plugin/task/file/LogFileTask.java | 5 +
.../inlong/agent/plugin/task/TestLogFileTask.java | 5 +-
inlong-audit/audit-docker/audit-docker.sh | 4 +-
inlong-audit/audit-sdk/README.md | 61 ++-
.../java/org/apache/inlong/audit/AuditIdEnum.java | 138 +++++
.../org/apache/inlong/audit/AuditReporterImpl.java | 84 ++-
.../java/org/apache/inlong/audit/MetricIdEnum.java | 72 +++
.../inlong/audit/entity/AuditInformation.java | 20 +-
.../org/apache/inlong/audit/entity/AuditType.java} | 40 +-
.../org/apache/inlong/audit/entity/FlowType.java} | 32 +-
.../exceptions/AuditTypeNotExistException.java | 13 +-
.../inlong/audit/util/AuditManagerUtils.java | 180 +++++++
.../apache/inlong/audit/AuditReporterImplTest.java | 48 ++
.../inlong/audit/util/AuditManagerUtilsTest.java | 126 +++++
.../apache/inlong/audit/cache/RealTimeQuery.java | 15 +-
.../apache/inlong/audit/config/SqlConstants.java | 26 +-
.../apache/inlong/audit/service/ApiService.java | 4 +-
.../org/apache/inlong/audit/source/JdbcSource.java | 16 +-
.../org/apache/inlong/audit/utils/CacheUtils.java | 4 +
.../java/utils/CacheUtilsTest.java} | 34 +-
.../org/apache/inlong/audit/config/JdbcConfig.java | 16 +-
inlong-audit/conf/application.properties | 8 +-
.../apache/inlong/common/enums/IndicatorType.java | 13 +-
.../inlong/common/pojo/agent/DataConfig.java | 1 +
.../inlong/common/pojo/sort/SortClusterConfig.java | 69 +++
.../apache/inlong/common/pojo/sort/SortConfig.java | 60 +++
.../inlong/common/pojo/sort/SortTaskConfig.java | 79 +++
.../common/pojo/sort/dataflow/DataFlowConfig.java | 26 +-
.../pojo/sort/dataflow/dataType/KvConfig.java | 1 +
.../pojo/sort/dataflow/sink/KafkaSinkConfig.java | 17 +-
.../common/pojo/sort/dataflow/sink/SinkConfig.java | 1 +
.../common/pojo/sort/mq/MqClusterConfig.java | 32 +-
.../KvConfig.java => node/KafkaNodeConfig.java} | 10 +-
.../inlong/common/pojo/sort/node/NodeConfig.java | 3 +
.../org/apache/inlong/common/util/ListUtil.java | 121 +++++
.../apache/inlong/common/util/SortConfigUtil.java | 287 ++++++++++
inlong-dashboard/src/core/utils/pattern.ts | 2 +
.../src/plugins/sources/defaults/File.ts | 32 +-
.../src/plugins/sources/defaults/PostgreSQL.ts | 110 ++++
.../src/ui/components/MultiSelectWithAll/index.tsx | 50 ++
inlong-dashboard/src/ui/locales/cn.json | 10 +-
inlong-dashboard/src/ui/locales/en.json | 8 +
.../src/ui/pages/Clusters/NodeEditModal.tsx | 69 ++-
.../src/ui/pages/GroupDetail/Audit/config.tsx | 6 +-
.../pages/GroupDetail/DataSources/DetailModal.tsx | 4 +
.../ui/pages/GroupDetail/OperationLog/config.tsx | 6 +
.../src/ui/pages/ModuleAudit/IdModule/config.tsx | 4 +-
.../src/ui/pages/ModuleAudit/IpModule/config.tsx | 4 +-
.../inlong/dataproxy/metrics/audit/AuditUtils.java | 33 +-
.../sink/mq/MessageQueueZoneSinkContext.java | 5 +-
.../apache/inlong/dataproxy/source/BaseSource.java | 2 +-
.../client/api/inner/ClientFactoryTest.java | 2 +-
.../inlong/manager/common/consts/SinkType.java | 4 +-
.../common/fieldtype/FieldTypeMappingReader.java | 13 +-
.../strategy/ClickHouseFieldTypeStrategy.java | 13 +-
.../strategy/DefaultFieldTypeStrategy.java | 43 +-
.../strategy/FieldTypeMappingStrategy.java | 13 +-
.../strategy/FieldTypeStrategyFactory.java} | 43 +-
.../strategy/IcebergFieldTypeStrategy.java | 15 +-
.../strategy/MongoDBFieldTypeStrategy.java | 15 +-
.../fieldtype/strategy/MySQLFieldTypeStrategy.java | 15 +-
.../strategy/OracleFieldTypeStrategy.java | 15 +-
.../strategy/PostgreSQLFieldTypeStrategy.java | 15 +-
.../strategy/SQLServerFieldTypeStrategy.java | 15 +-
.../resources/starrocks-field-type-mapping.yaml | 231 ++++++++
.../dao/mapper/InlongGroupEntityMapper.java | 3 +
.../dao/mapper/StreamSourceEntityMapper.java | 5 +
.../resources/mappers/AuditBaseEntityMapper.xml | 72 ---
.../mappers/InlongClusterNodeEntityMapper.xml | 7 +-
.../resources/mappers/InlongGroupEntityMapper.xml | 9 +
.../resources/mappers/StreamSourceEntityMapper.xml | 17 +-
inlong-manager/manager-docker/Dockerfile | 1 +
inlong-manager/manager-docker/manager-docker.sh | 5 +-
.../plugin/listener/StartupSortListener.java | 101 +---
.../plugin/listener/StartupStreamListener.java | 97 +---
.../inlong/manager/plugin/util/FlinkUtils.java | 94 ++++
.../manager/pojo/consume/BriefMQMessage.java | 3 +
.../manager/pojo/node/kafka/KafkaDataNodeDTO.java | 2 +-
.../manager/pojo/node/kafka/KafkaDataNodeInfo.java | 2 +-
.../manager/pojo/sink/ck/ClickHouseSinkDTO.java | 9 +-
.../manager/pojo/sink/doris/DorisSinkDTO.java | 10 +-
.../pojo/sink/es/ElasticsearchFieldInfo.java | 2 +-
.../manager/pojo/sink/es/ElasticsearchSink.java | 4 +-
.../pojo/sink/es/ElasticsearchSinkRequest.java | 2 +-
.../inlong/manager/pojo/sink/hive/HiveSinkDTO.java | 10 +-
.../pojo/sink/postgresql/PostgreSQLSinkDTO.java | 9 +-
.../pojo/sink/starrocks/StarRocksSinkDTO.java | 10 +-
.../pojo/sort/node/ExtractNodeProviderFactory.java | 40 +-
.../pojo/sort/node/LoadNodeProviderFactory.java | 53 +-
.../inlong/manager/pojo/sort/node/NodeFactory.java | 30 +-
.../sort/node/provider/ClickHouseProvider.java | 15 +-
.../pojo/sort/node/provider/DorisProvider.java | 2 +
.../sort/node/provider/ElasticsearchProvider.java | 5 +-
.../pojo/sort/node/provider/GreenplumProvider.java | 3 +
.../pojo/sort/node/provider/HBaseProvider.java | 2 +
.../pojo/sort/node/provider/HDFSProvider.java | 2 +
.../pojo/sort/node/provider/HiveProvider.java | 2 +
.../pojo/sort/node/provider/HudiProvider.java | 3 +
.../pojo/sort/node/provider/IcebergProvider.java | 16 +-
.../pojo/sort/node/provider/KafkaProvider.java | 2 +
.../pojo/sort/node/provider/KuduProvider.java | 3 +
.../pojo/sort/node/provider/MongoDBProvider.java | 14 +-
.../sort/node/provider/MySQLBinlogProvider.java | 12 +-
.../pojo/sort/node/provider/MySQLProvider.java | 12 +-
.../pojo/sort/node/provider/OracleProvider.java | 16 +-
.../sort/node/provider/PostgreSQLProvider.java | 17 +-
.../pojo/sort/node/provider/PulsarProvider.java | 7 +-
.../pojo/sort/node/provider/RedisProvider.java | 2 +
.../pojo/sort/node/provider/SQLServerProvider.java | 17 +-
.../pojo/sort/node/provider/StarRocksProvider.java | 2 +
.../node/provider/TDSQLPostgreSQLProvider.java | 3 +
.../pojo/sort/node/provider/TubeMqProvider.java | 3 +
.../manager/pojo/sort/util/FieldInfoUtils.java | 6 +-
.../manager/pojo/source/DataAddTaskRequest.java | 3 +
.../inlong/manager/pojo/source/SourceRequest.java | 3 +
.../inlong/manager/pojo/source/StreamSource.java | 3 +
.../manager/pojo/source/file/FileSource.java | 3 -
.../manager/pojo/source/file/FileSourceDTO.java | 4 +-
.../pojo/source/file/FileSourceRequest.java | 3 -
.../manager/pojo/stream/InlongStreamInfo.java | 7 +-
.../manager/pojo/stream/InlongStreamRequest.java | 7 +-
inlong-manager/manager-service/pom.xml | 11 +
.../service/cluster/ClusterConfigService.java | 14 +-
.../service/cluster/ClusterConfigServiceImpl.java | 87 ++++
.../service/cluster/InlongClusterServiceImpl.java | 8 +
.../inlong/manager/service/core/AuditService.java | 4 +-
.../service/core/impl/AgentServiceImpl.java | 20 +-
.../service/core/impl/AuditServiceImpl.java | 68 ++-
.../manager/service/group/InlongGroupService.java | 28 +-
.../service/group/InlongGroupServiceImpl.java | 93 ++--
.../service/heartbeat/HeartbeatManager.java | 33 +-
.../listener/queue/ClusterConfigListener.java | 64 +--
.../message/InlongMsgDeserializeOperator.java | 1 +
.../service/node/kafka/KafkaDataNodeOperator.java | 10 +
.../resource/queue/pulsar/PulsarOperator.java | 7 +-
.../queue/pulsar/PulsarQueueResourceOperator.java | 14 +-
.../queue/pulsar/QueryLatestMessagesRunnable.java | 8 +-
.../sink/es/ElasticsearchResourceOperator.java | 2 +-
.../sink/starrocks/StarRocksSqlBuilder.java | 10 +-
.../resource/sort/SortFlinkConfigOperator.java | 11 +-
.../manager/service/sink/AbstractSinkOperator.java | 31 ++
.../manager/service/sink/StreamSinkOperator.java | 9 +
.../service/sink/es/ElasticsearchSinkOperator.java | 4 +-
.../service/sink/kafka/KafkaSinkOperator.java | 22 +
.../service/source/StreamSourceService.java | 9 +
.../service/source/StreamSourceServiceImpl.java | 13 +
.../service/stream/InlongStreamServiceImpl.java | 14 +-
.../service/tenant/InlongTenantServiceImpl.java | 31 +-
.../inlong/manager/service/ServiceBaseTest.java | 2 +
.../service/sink/ElasticsearchSinkServiceTest.java | 2 +-
.../main/resources/h2/apache_inlong_manager.sql | 54 --
.../manager-web/sql/apache_inlong_manager.sql | 68 ---
.../manager/web/controller/AuditController.java | 4 +-
.../web/controller/ClusterConfigController.java | 53 ++
.../web/controller/InlongGroupController.java | 7 +
.../web/controller/StreamSourceController.java | 12 +-
.../openapi/OpenInLongGroupController.java | 4 +-
.../dataproxy-sdk-golang/connpool/connpool.go | 24 +-
.../dataproxy-sdk-golang/dataproxy/client.go | 12 +-
.../dataproxy-sdk-golang/go.mod | 28 +-
.../dataproxy-sdk-golang/go.sum | 77 +--
.../apache/inlong/sdk/sort/entity/InLongTopic.java | 6 +-
.../sort/fetcher/tube/TubeSingleTopicFetcher.java | 2 +-
.../sdk/sort/impl/QueryConsumeConfigImpl.java | 4 +-
.../config/holder/v2/SortConfigHolder.java | 5 +-
.../loader/ClassResourceQueryConsumeConfig.java | 3 +-
.../v2/ClassResourceSortClusterConfigLoader.java | 5 +-
.../sort/standalone/sink/cls/ClsSinkContext.java | 5 +-
.../standalone/sink/elasticsearch/EsIdConfig.java | 196 ++-----
.../sort/standalone/sink/elasticsearch/EsSink.java | 2 +-
.../sink/elasticsearch/EsSinkContext.java | 70 +--
.../sink/kafka/KafkaFederationSinkContext.java | 70 +--
.../sort/standalone/sink/kafka/KafkaIdConfig.java | 141 +----
.../sink/kafka/KafkaProducerCluster.java | 27 +-
.../sink/kafka/KafkaProducerFederation.java | 99 +---
.../sink/pulsar/PulsarFederationSinkContext.java | 105 +---
.../sink/pulsar/PulsarFederationWorker.java | 29 --
.../standalone/sink/pulsar/PulsarIdConfig.java | 151 +-----
.../sink/pulsar/PulsarProducerCluster.java | 36 +-
.../sink/pulsar/PulsarProducerFederation.java | 85 +--
.../src/test/java/SortClusterConfig.conf | 193 ++++---
.../src/test/resources/SortClusterConfig.conf | 193 ++++---
.../inlong/sort/protocol/node/format/Format.java | 3 +-
.../inlong/sort/protocol/node/format/KvFormat.java | 131 +++++
.../sort/protocol/node/format/KvFormatTest.java | 14 +-
inlong-sort/sort-dist/pom.xml | 2 +-
.../sort-flink-v1.13/sort-connectors/hive/pom.xml | 1 -
.../sort-flink-v1.15/sort-connectors/hudi/pom.xml | 1 -
.../inlong/sort/hudi/sink/StreamWriteFunction.java | 495 ++++++++++++++++++
.../inlong/sort/hudi/sink/StreamWriteOperator.java | 43 ++
.../sort/hudi/sink/append/AppendWriteFunction.java | 156 ++++++
.../sort/hudi/sink/append/AppendWriteOperator.java | 44 ++
.../sink/bucket/BucketStreamWriteFunction.java | 185 +++++++
.../sink/bucket/BucketStreamWriteOperator.java | 36 +-
.../hudi/sink/bulk/BulkInsertWriteFunction.java | 229 ++++++++
.../hudi/sink/bulk/BulkInsertWriteOperator.java | 55 ++
.../inlong/sort/hudi/sink/utils/Pipelines.java | 469 +++++++++++++++++
.../inlong/sort/hudi/table/HoodieTableFactory.java | 355 +++++++++++++
.../inlong/sort/hudi/table/HoodieTableSink.java | 150 ++++++
.../hudi/table/sink/HudiTableInlongFactory.java | 55 --
.../org.apache.flink.table.factories.Factory | 2 +-
.../sort-connectors/iceberg/pom.xml | 6 +-
.../inlong/sort/mongodb/DebeziumChangeFetcher.java | 322 ++++++++++++
.../mongodb/DebeziumDeserializationSchema.java | 42 ++
.../sort/mongodb/DebeziumSourceFunction.java | 563 ++++++++++++++++++++
.../MongoDBConnectorDeserializationSchema.java | 12 +-
.../apache/inlong/sort/mongodb/MongoDBSource.java | 344 ++++++++++++
.../inlong/sort/mongodb/MongoDBTableSource.java | 9 +-
.../sort/mongodb/source/IncrementalSource.java | 217 ++++++++
.../source/IncrementalSourceRecordEmitter.java | 176 +++++++
.../sort/mongodb/source/MongoDBRecordEmitter.java | 111 ++++
.../inlong/sort/mongodb/source/MongoDBSource.java | 95 ++++
.../sort/mongodb/source/MongoDBSourceBuilder.java | 201 +++++++
.../sort-connectors/mysql-cdc/pom.xml | 1 +
.../apache/inlong/sort/mysql/MySqlTableSource.java | 371 +++++++++++++
.../inlong/sort/mysql/MysqlTableFactory.java | 18 +-
.../mysql}/RowDataDebeziumDeserializeSchema.java | 46 +-
.../inlong/sort/mysql/source/MySqlSource.java | 239 +++++++++
.../sort/mysql/source/MySqlSourceBuilder.java | 246 +++++++++
.../inlong/sort/postgre/DebeziumChangeFetcher.java | 322 ++++++++++++
.../postgre/DebeziumDeserializationSchema.java | 42 ++
.../sort/postgre/DebeziumSourceFunction.java | 579 +++++++++++++++++++++
.../inlong/sort/postgre/PostgreSQLSource.java | 184 +++++++
.../inlong/sort/postgre/PostgreSQLTableSource.java | 9 +-
.../postgre/RowDataDebeziumDeserializeSchema.java | 27 +-
.../sort/sqlserver/DebeziumChangeFetcher.java | 322 ++++++++++++
.../sqlserver/DebeziumDeserializationSchema.java | 42 ++
.../sort/sqlserver/DebeziumSourceFunction.java | 565 ++++++++++++++++++++
.../RowDataDebeziumDeserializeSchema.java | 24 +-
.../inlong/sort/sqlserver/SqlServerSource.java | 160 ++++++
.../sort/sqlserver/SqlServerTableSource.java | 6 +-
.../sort/sqlserver/SqlserverTableFactory.java | 6 +-
.../inlong/sort/formats/base/TableFormatUtils.java | 16 +
.../InLongMsgTlogCsvFormatDeserializer.java | 24 +-
.../InLongMsgTlogCsvMixedFormatDeserializer.java | 11 +-
.../inlongmsgtlogcsv/InLongMsgTlogCsvUtils.java | 5 +-
.../InLongMsgTlogCsvFormatDeserializerTest.java | 10 +-
.../sort/formats/inlongmsg/InLongMsgUtils.java | 4 +-
.../InLongMsgCsvFormatDeserializerTest.java | 4 +
.../InLongMsgTlogCsvFormatDeserializer.java | 23 +-
.../inlongmsgtlogcsv/InLongMsgTlogCsvUtils.java | 5 +-
.../InLongMsgTlogCsvFormatDeserializerTest.java | 4 +-
.../inlong/sort/formats/kv/KvFormatFactory.java | 6 +-
.../inlong/tubemq/corerpc/utils/MixUtils.java | 2 +-
licenses/inlong-manager/LICENSE | 1 -
licenses/inlong-manager/NOTICE | 2 -
licenses/inlong-sort-connectors/LICENSE | 65 ++-
licenses/inlong-sort-standalone/LICENSE | 2 +-
pom.xml | 3 +-
275 files changed, 11787 insertions(+), 2607 deletions(-)
delete mode 100644
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/CommandDb.java
create mode 100755
inlong-agent/agent-common/src/test/java/org/apache/inlong/agent/metrics/TestAuditUtils.java
copy
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/DefaultFieldTypeStrategy.java
=>
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/instance/PostgreSQLInstance.java
(71%)
copy
inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/{PulsarTask.java
=> PostgreSQLTask.java} (53%)
create mode 100644
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/AuditIdEnum.java
create mode 100644
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/MetricIdEnum.java
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/SortConfig.java
=>
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/entity/AuditInformation.java
(72%)
copy
inlong-audit/{audit-service/src/main/java/org/apache/inlong/audit/utils/CacheUtils.java
=> audit-sdk/src/main/java/org/apache/inlong/audit/entity/AuditType.java} (57%)
copy
inlong-audit/{audit-service/src/main/java/org/apache/inlong/audit/utils/CacheUtils.java
=> audit-sdk/src/main/java/org/apache/inlong/audit/entity/FlowType.java} (61%)
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/dataType/KvConfig.java
=>
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/exceptions/AuditTypeNotExistException.java
(77%)
create mode 100644
inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditManagerUtils.java
create mode 100644
inlong-audit/audit-sdk/src/test/java/org/apache/inlong/audit/AuditReporterImplTest.java
create mode 100644
inlong-audit/audit-sdk/src/test/java/org/apache/inlong/audit/util/AuditManagerUtilsTest.java
copy
inlong-audit/audit-service/src/{main/java/org/apache/inlong/audit/utils/CacheUtils.java
=> test/java/utils/CacheUtilsTest.java} (59%)
rename
inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/AuditBaseEntity.java
=>
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/sink/KafkaSinkConfig.java
(73%)
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/{dataflow/dataType/KvConfig.java
=> node/KafkaNodeConfig.java} (80%)
create mode 100644
inlong-common/src/main/java/org/apache/inlong/common/util/ListUtil.java
create mode 100644
inlong-common/src/main/java/org/apache/inlong/common/util/SortConfigUtil.java
create mode 100644
inlong-dashboard/src/ui/components/MultiSelectWithAll/index.tsx
rename
inlong-manager/{manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/AuditBaseEntityMapper.java
=>
manager-common/src/main/java/org/apache/inlong/manager/common/fieldtype/strategy/FieldTypeStrategyFactory.java}
(54%)
create mode 100644
inlong-manager/manager-common/src/main/resources/starrocks-field-type-mapping.yaml
delete mode 100644
inlong-manager/manager-dao/src/main/resources/mappers/AuditBaseEntityMapper.xml
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/dataType/KvConfig.java
=>
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/cluster/ClusterConfigService.java
(77%)
create mode 100644
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/cluster/ClusterConfigServiceImpl.java
create mode 100644
inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/ClusterConfigController.java
create mode 100644
inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/format/KvFormat.java
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/dataType/KvConfig.java
=>
inlong-sort/sort-common/src/test/java/org/apache/inlong/sort/protocol/node/format/KvFormatTest.java
(75%)
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/sink/StreamWriteFunction.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/sink/StreamWriteOperator.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/sink/append/AppendWriteFunction.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/sink/append/AppendWriteOperator.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/sink/bucket/BucketStreamWriteFunction.java
rename
inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/audit/AuditBaseResponse.java
=>
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/sink/bucket/BucketStreamWriteOperator.java
(50%)
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/sink/bulk/BulkInsertWriteFunction.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/sink/bulk/BulkInsertWriteOperator.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/sink/utils/Pipelines.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/table/HoodieTableFactory.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/table/HoodieTableSink.java
delete mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/hudi/src/main/java/org/apache/inlong/sort/hudi/table/sink/HudiTableInlongFactory.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/mongodb/DebeziumChangeFetcher.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/mongodb/DebeziumDeserializationSchema.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/mongodb/DebeziumSourceFunction.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/mongodb/MongoDBSource.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/mongodb/source/IncrementalSource.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/mongodb/source/IncrementalSourceRecordEmitter.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/mongodb/source/MongoDBRecordEmitter.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/mongodb/source/MongoDBSource.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/mongodb/source/MongoDBSourceBuilder.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/mysql/MySqlTableSource.java
copy
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/{postgres-cdc/src/main/java/org/apache/inlong/sort/postgre
=>
mysql-cdc/src/main/java/org/apache/inlong/sort/mysql}/RowDataDebeziumDeserializeSchema.java
(95%)
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/mysql/source/MySqlSource.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/mysql/source/MySqlSourceBuilder.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/postgres-cdc/src/main/java/org/apache/inlong/sort/postgre/DebeziumChangeFetcher.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/postgres-cdc/src/main/java/org/apache/inlong/sort/postgre/DebeziumDeserializationSchema.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/postgres-cdc/src/main/java/org/apache/inlong/sort/postgre/DebeziumSourceFunction.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/postgres-cdc/src/main/java/org/apache/inlong/sort/postgre/PostgreSQLSource.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/sqlserver-cdc/src/main/java/org/apache/inlong/sort/sqlserver/DebeziumChangeFetcher.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/sqlserver-cdc/src/main/java/org/apache/inlong/sort/sqlserver/DebeziumDeserializationSchema.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/sqlserver-cdc/src/main/java/org/apache/inlong/sort/sqlserver/DebeziumSourceFunction.java
create mode 100644
inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/sqlserver-cdc/src/main/java/org/apache/inlong/sort/sqlserver/SqlServerSource.java
