This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch auto-pick-58898-branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from b90df915068 branch-4.0: [fix](inverted index) also release dir when
search cache after writer enable and fix get_index_file_names for v1 #59139
(#59156)
add 7ce1ae375a5 [Feature](Streaming Job) Extend streaming job to support
MySQL synchronization (#58898)
No new revisions were added by this update.
Summary of changes:
be/src/common/config.cpp | 5 +
be/src/common/config.h | 6 +
be/src/runtime/cdc_client_mgr.cpp | 269 ++++++++
be/src/runtime/cdc_client_mgr.h | 63 ++
be/src/runtime/exec_env.h | 3 +
be/src/runtime/exec_env_init.cpp | 3 +
be/src/service/internal_service.cpp | 15 +
be/src/service/internal_service.h | 5 +
be/test/runtime/cdc_client_mgr_test.cpp | 641 +++++++++++++++++
build.sh | 29 +-
docker/thirdparties/docker-compose/mysql/my.cnf | 3 +
.../main/java/org/apache/doris/common/Config.java | 6 +
.../apache/doris/job/cdc/DataSourceConfigKeys.java | 35 +
.../job/cdc/request/CompareOffsetRequest.java} | 38 +-
.../doris/job/cdc/request/FetchRecordRequest.java} | 34 +-
.../job/cdc/request/FetchTableSplitsRequest.java} | 35 +-
.../doris/job/cdc/request/JobBaseConfig.java} | 33 +-
.../job/cdc/request/JobBaseRecordRequest.java} | 31 +-
.../doris/job/cdc/request/WriteRecordRequest.java} | 37 +-
.../doris/job/cdc/split/AbstractSourceSplit.java} | 32 +-
.../apache/doris/job/cdc/split/BinlogSplit.java} | 45 +-
.../apache/doris/job/cdc/split/SnapshotSplit.java | 62 ++
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 16 +-
.../doris/datasource/jdbc/client/JdbcClient.java | 24 +
.../datasource/jdbc/client/JdbcMySQLClient.java | 21 +
.../org/apache/doris/httpv2/rest/LoadAction.java | 13 -
.../doris/httpv2/rest/RestBaseController.java | 13 +
.../doris/httpv2/rest/StreamingJobAction.java | 89 +++
.../DataSourceType.java} | 26 +-
.../insert/streaming/AbstractStreamingTask.java | 166 +++++
.../streaming/DataSourceConfigValidator.java | 79 +++
.../insert/streaming/StreamingInsertJob.java | 450 ++++++++++--
.../insert/streaming/StreamingInsertTask.java | 134 +---
.../streaming/StreamingJobSchedulerTask.java | 10 +-
.../insert/streaming/StreamingJobStatistic.java | 3 +
.../insert/streaming/StreamingMultiTblTask.java | 282 ++++++++
.../org/apache/doris/job/manager/JobManager.java | 27 +-
.../doris/job/manager/StreamingTaskManager.java | 14 +-
.../doris/job/offset/SourceOffsetProvider.java | 23 +
.../apache/doris/job/offset/jdbc/JdbcOffset.java | 77 +++
.../job/offset/jdbc/JdbcSourceOffsetProvider.java | 494 ++++++++++++++
.../job/scheduler/StreamingTaskScheduler.java | 14 +-
.../apache/doris/job/util/StreamingJobUtils.java | 337 +++++++++
.../doris/nereids/parser/LogicalPlanBuilder.java | 37 +-
.../trees/plans/commands/AlterJobCommand.java | 130 +++-
.../trees/plans/commands/CreateJobCommand.java | 8 +
.../trees/plans/commands/info/CreateJobInfo.java | 97 ++-
.../org/apache/doris/rpc/BackendServiceClient.java | 5 +
.../org/apache/doris/rpc/BackendServiceProxy.java | 11 +
.../doris/tablefunction/MetadataGenerator.java | 6 +-
fs_brokers/cdc_client/build.sh | 36 +
fs_brokers/cdc_client/pom.xml | 261 +++++++
.../doris/cdcclient/CdcClientApplication.java | 40 ++
.../apache/doris/cdcclient/common/Constants.java | 26 +-
.../org/apache/doris/cdcclient/common/Env.java | 126 ++++
.../doris/cdcclient/config/BackendPortHolder.java | 32 +-
.../cdcclient/config/GlobalExceptionHandler.java | 34 +-
.../cdcclient/controller/ClientController.java | 106 +++
.../cdcclient/exception/CdcClientException.java | 27 +-
.../cdcclient/exception/StreamLoadException.java | 40 +-
.../cdcclient/model/response/RecordWithMeta.java | 32 +-
.../doris/cdcclient/model/rest/RestResponse.java | 54 ++
.../cdcclient/service/PipelineCoordinator.java | 224 ++++++
.../cdcclient/sink/BatchBufferHttpEntity.java | 77 +++
.../doris/cdcclient/sink/BatchBufferStream.java | 73 ++
.../doris/cdcclient/sink/BatchRecordBuffer.java | 146 ++++
.../doris/cdcclient/sink/DorisBatchStreamLoad.java | 543 +++++++++++++++
.../doris/cdcclient/sink/HttpPutBuilder.java | 135 ++++
.../apache/doris/cdcclient/sink/RespContent.java | 170 +++++
.../deserialize/DebeziumJsonDeserializer.java | 310 +++++++++
.../deserialize/SourceRecordDeserializer.java | 28 +-
.../doris/cdcclient/source/factory/DataSource.java | 26 +-
.../source/factory/SourceReaderFactory.java | 58 ++
.../cdcclient/source/reader/SourceReader.java | 74 ++
.../cdcclient/source/reader/SplitReadResult.java | 33 +-
.../cdcclient/source/reader/SplitRecords.java | 60 ++
.../source/reader/mysql/MySqlSourceReader.java | 756 +++++++++++++++++++++
.../apache/doris/cdcclient/utils/ConfigUtil.java | 201 ++++++
.../org/apache/doris/cdcclient/utils/HttpUtil.java | 55 ++
.../source/mocks/MockSplitEnumeratorContext.java | 320 +++++++++
.../src/main/resources/application.properties | 20 +
.../cdc_client/src/main/resources/log4j.properties | 32 +
gensrc/proto/internal_service.proto | 11 +
.../streaming_job/cdc/test_streaming_mysql_job.out | 18 +
.../cdc/test_streaming_mysql_job_all_type.out | 54 ++
.../cdc/test_streaming_mysql_job_exclude.out | 5 +
.../cdc/test_streaming_mysql_job_priv.out | 6 +
.../cdc/test_streaming_mysql_job.groovy | 167 +++++
.../cdc/test_streaming_mysql_job_all_type.groovy | 156 +++++
.../test_streaming_mysql_job_create_alter.groovy | 464 +++++++++++++
.../cdc/test_streaming_mysql_job_dup.groovy | 79 +++
.../cdc/test_streaming_mysql_job_exclude.groovy | 117 ++++
.../cdc/test_streaming_mysql_job_priv.groovy | 202 ++++++
.../cdc/test_streaming_mysql_job_restart_fe.groovy | 126 ++++
94 files changed, 8983 insertions(+), 613 deletions(-)
create mode 100644 be/src/runtime/cdc_client_mgr.cpp
create mode 100644 be/src/runtime/cdc_client_mgr.h
create mode 100644 be/test/runtime/cdc_client_mgr_test.cpp
create mode 100644
fe/fe-common/src/main/java/org/apache/doris/job/cdc/DataSourceConfigKeys.java
copy
fe/{fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fe-common/src/main/java/org/apache/doris/job/cdc/request/CompareOffsetRequest.java}
(55%)
copy
fe/{fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fe-common/src/main/java/org/apache/doris/job/cdc/request/FetchRecordRequest.java}
(64%)
copy
fe/{fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fe-common/src/main/java/org/apache/doris/job/cdc/request/FetchTableSplitsRequest.java}
(59%)
copy
fe/{fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fe-common/src/main/java/org/apache/doris/job/cdc/request/JobBaseConfig.java}
(63%)
copy
fe/{fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fe-common/src/main/java/org/apache/doris/job/cdc/request/JobBaseRecordRequest.java}
(67%)
copy
fe/{fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fe-common/src/main/java/org/apache/doris/job/cdc/request/WriteRecordRequest.java}
(61%)
copy
fe/{fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fe-common/src/main/java/org/apache/doris/job/cdc/split/AbstractSourceSplit.java}
(66%)
copy
fe/{fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=> fe-common/src/main/java/org/apache/doris/job/cdc/split/BinlogSplit.java}
(53%)
create mode 100644
fe/fe-common/src/main/java/org/apache/doris/job/cdc/split/SnapshotSplit.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StreamingJobAction.java
copy
fe/fe-core/src/main/java/org/apache/doris/job/{extensions/insert/streaming/StreamingJobStatistic.java
=> common/DataSourceType.java} (63%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/AbstractStreamingTask.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/DataSourceConfigValidator.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcOffset.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/job/util/StreamingJobUtils.java
create mode 100755 fs_brokers/cdc_client/build.sh
create mode 100644 fs_brokers/cdc_client/pom.xml
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/CdcClientApplication.java
copy
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/common/Constants.java
(63%)
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/common/Env.java
copy
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/config/BackendPortHolder.java
(64%)
copy
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/config/GlobalExceptionHandler.java
(54%)
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/controller/ClientController.java
copy
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/exception/CdcClientException.java
(64%)
copy
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/exception/StreamLoadException.java
(54%)
copy
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/model/response/RecordWithMeta.java
(64%)
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/model/rest/RestResponse.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/service/PipelineCoordinator.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/sink/BatchBufferHttpEntity.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/sink/BatchBufferStream.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/sink/BatchRecordBuffer.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/sink/DorisBatchStreamLoad.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/sink/HttpPutBuilder.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/sink/RespContent.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/deserialize/DebeziumJsonDeserializer.java
copy
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/deserialize/SourceRecordDeserializer.java
(63%)
copy
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/factory/DataSource.java
(63%)
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/factory/SourceReaderFactory.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/SourceReader.java
copy
fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingJobStatistic.java
=>
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/SplitReadResult.java
(62%)
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/SplitRecords.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/reader/mysql/MySqlSourceReader.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/utils/ConfigUtil.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/utils/HttpUtil.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/flink/api/connector/source/mocks/MockSplitEnumeratorContext.java
create mode 100644
fs_brokers/cdc_client/src/main/resources/application.properties
create mode 100644 fs_brokers/cdc_client/src/main/resources/log4j.properties
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_all_type.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_exclude.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_priv.out
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_all_type.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_create_alter.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_dup.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_exclude.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_priv.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_restart_fe.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
