This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
from c3065e33a4e [fix](file cache) Fix peer IO profile statistics diff
(#69000)
new a6542f220c2 branch-4.1: [fix](fe) Persist routine load job before
scheduling #67681- #68560 (#68773)
new 2c8844598db [fix](nereids) Fix dead visitor dispatch of
PhysicalLazyMaterializeTVFScan (pick #68871) (#69006)
new 12f8ed61174 [fix](cloud) set enable_cloud_colocate_consistent_hash
default value to false (#68065) (#68946)
new 0212698619b [fix](fe) Report active compute group in processlist
(#68111) (#68948)
new b56c8d19fdc branch-4.1: [fix](cloud) Preserve transaction metadata on
precommit replay #69005 (#69022)
new a8b15c1ff47 branch-4.1: [feature](fe) Support S3 notification-based
streaming ingestion #68876 (#68992)
new 0c77b73d15d branch-4.1: [opt](partition) Add global dynamic partition
drop protection #68844 (#69065)
new 80583d60057 branch-4.1: [fix](mv rewrite) Return 0 for a COUNT rollup
that hits a materialized view #68863 (#69072)
new 84cdbc7112f branch-4.1: [improvement](streaming-job) Improve CDC
snapshot, lifecycle and schema changes #68432 #68270 #68616 (#69067)
new 51d4b709b7a branch-4.1: [fix](load) Reject malformed numeric HTTP
headers instead of killing BE #68462 (#69081)
new 7c330d2c361 branch-4.1: [fix](be) Preserve the sample triggering
QuantileState digest conversion #69037 (#69049)
new 7b81582c976 branch-4.1: [fix](cloud) Preserve recreated compute group
mapping #67913 (#68939)
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/core/value/quantile_state.cpp | 2 +-
be/src/service/http/action/http_stream.cpp | 18 +-
be/src/service/http/action/jeprofile_actions.cpp | 12 +-
be/src/service/http/action/stream_load_2pc.cpp | 3 +
be/src/service/http/utils.cpp | 11 +-
be/src/util/string_util.cpp | 12 +
be/src/util/string_util.h | 2 +
be/src/util/tdigest.h | 3 +-
be/test/core/value/quantile_state_test.cpp | 60 +
be/test/service/http/http_utils_test.cpp | 22 +
be/test/service/http/stream_load_test.cpp | 36 +
be/test/util/string_util_test.cpp | 15 +
be/test/util/tdigest_test.cpp | 29 +
cloud/src/meta-service/meta_service_txn.cpp | 2 +
cloud/test/meta_service_test.cpp | 79 ++
.../main/java/org/apache/doris/common/Config.java | 6 +-
.../apache/doris/job/cdc/DataSourceConfigKeys.java | 2 +
.../doris/job/cdc/request/TaskFailureRequest.java | 2 +
.../doris/job/cdc/request/WriteRecordRequest.java | 2 +
fe/fe-core/pom.xml | 4 +
.../doris/clone/DynamicPartitionScheduler.java | 30 +-
.../doris/cloud/catalog/CloudClusterChecker.java | 25 +-
.../cloud/catalog/CloudInstanceStatusChecker.java | 59 +-
.../apache/doris/cloud/catalog/CloudReplica.java | 2 +-
.../doris/cloud/system/CloudSystemInfoService.java | 74 +-
.../java/org/apache/doris/common/util/S3Util.java | 39 +-
.../org/apache/doris/job/common/FailureReason.java | 8 +
.../doris/job/executor/DispatchTaskHandler.java | 7 +-
.../streaming/DataSourceConfigValidator.java | 7 +-
.../insert/streaming/StreamingInsertJob.java | 185 +--
.../insert/streaming/StreamingInsertTask.java | 36 +-
.../insert/streaming/StreamingJobProperties.java | 56 +-
.../insert/streaming/StreamingMultiTblTask.java | 34 +-
.../org/apache/doris/job/manager/JobManager.java | 1 -
.../doris/job/offset/SourceOffsetProvider.java | 24 +-
.../job/offset/SourceOffsetProviderFactory.java | 6 +-
.../job/offset/jdbc/JdbcSourceOffsetProvider.java | 59 +-
.../offset/jdbc/JdbcTvfSourceOffsetProvider.java | 7 +-
.../s3/{S3Offset.java => S3EventOffset.java} | 58 +-
.../job/offset/s3/S3EventSourceOffsetProvider.java | 879 +++++++++++++++
.../org/apache/doris/job/offset/s3/S3Offset.java | 8 +-
.../job/offset/s3/S3SourceOffsetProvider.java | 91 +-
.../doris/load/routineload/RoutineLoadManager.java | 3 +-
.../java/org/apache/doris/metric/MetricRepo.java | 19 +-
.../nereids/rules/rewrite/SetPreAggStatus.java | 20 +
.../doris/nereids/stats/ExpressionEstimation.java | 7 +
.../trees/expressions/functions/agg/Count.java | 3 +-
.../trees/expressions/functions/table/S3.java | 30 +-
.../trees/plans/commands/AlterJobCommand.java | 30 +-
.../commands/insert/InsertIntoTableCommand.java | 10 +
.../physical/PhysicalLazyMaterializeTVFScan.java | 2 +-
.../java/org/apache/doris/qe/ConnectContext.java | 8 +-
.../java/org/apache/doris/qe/SessionVariable.java | 17 +
.../main/java/org/apache/doris/qe/VariableMgr.java | 30 +
.../CdcStreamTableValuedFunction.java | 4 +-
.../ExternalFileTableValuedFunction.java | 8 +
.../doris/tablefunction/S3TableValuedFunction.java | 11 +-
.../doris/catalog/DynamicPartitionTableTest.java | 97 ++
.../cloud/catalog/CloudClusterCheckerTest.java | 308 +++++
.../cloud/catalog/CloudColocatePlacementTest.java | 1 +
.../cloud/system/CloudSystemInfoServiceTest.java | 44 +
.../org/apache/doris/common/util/S3UtilTest.java | 55 +-
.../streaming/DataSourceConfigValidatorTest.java | 38 +
.../streaming/StreamingInsertJobLagTest.java | 25 +-
.../StreamingInsertJobOffsetPersistenceTest.java | 175 ++-
.../StreamingInsertJobStatusTransitionTest.java | 132 +++
.../streaming/StreamingInsertTaskAuditTest.java | 156 ++-
.../streaming/StreamingJobPropertiesTest.java | 65 +-
.../StreamingMultiTblTaskParallelismTest.java | 109 ++
.../JdbcSourceOffsetProviderAsyncSplitTest.java | 14 +
.../jdbc/JdbcSourceOffsetProviderOffsetTest.java | 102 ++
.../doris/job/offset/s3/S3EventOffsetTest.java | 56 +
.../offset/s3/S3EventSourceOffsetProviderTest.java | 1185 ++++++++++++++++++++
.../job/offset/s3/S3SourceOffsetProviderTest.java | 67 +-
.../routineload/RoutineLoadJobPersistenceTest.java | 132 +++
.../java/org/apache/doris/metric/MetricsTest.java | 29 +
.../nereids/rules/rewrite/SetPreAggStatusTest.java | 6 +
.../nereids/stats/ExpressionEstimationTest.java | 14 +
.../physical/PhysicalPlanVisitorDispatchTest.java | 177 +++
.../org/apache/doris/qe/ConnectContextTest.java | 18 +
.../java/org/apache/doris/qe/VariableMgrTest.java | 26 +
.../CdcStreamTableValuedFunctionTest.java | 28 +
.../tablefunction/S3TableValuedFunctionTest.java | 147 +++
.../cdcclient/service/PipelineCoordinator.java | 52 +-
.../deserialize/DebeziumJsonDeserializer.java | 8 +
.../source/deserialize/DeserializeResult.java | 64 +-
.../deserialize/MySqlDebeziumJsonDeserializer.java | 133 ++-
.../PostgresDebeziumJsonDeserializer.java | 154 +--
.../mysql/CustomAlterTableParserListener.java | 16 +-
.../source/parse/mysql/MySqlSchemaChange.java | 45 +-
.../source/reader/JdbcIncrementalSourceReader.java | 36 +-
.../source/reader/mysql/MySqlSourceReader.java | 36 +-
.../reader/postgres/PostgresSourceReader.java | 41 +-
.../doris/cdcclient/utils/SchemaChangeManager.java | 96 +-
.../cdcclient/itcase/CdcClientWriteHarness.java | 9 +
.../doris/cdcclient/itcase/MockDorisServer.java | 8 +
.../itcase/PostgresWriteSchemaChangeITCase.java | 29 +-
.../MySqlSchemaChangeDeserializeTest.java | 183 ++-
.../PostgresSchemaChangeDeserializeTest.java | 172 ++-
.../parse/mysql/CustomMySqlAntlrDdlParserTest.java | 28 +-
.../source/reader/mysql/MySqlSourceReaderTest.java | 40 +
.../cdcclient/utils/SchemaChangeManagerTest.java | 143 ++-
.../test_compute_group_name_reconciliation.out} | 24 +-
.../cdc/test_streaming_mysql_job_col_filter.out | 8 +
.../test_streaming_mysql_job_non_column_ddl.out | 52 +
.../cdc/test_streaming_mysql_job_sc_advanced.out | 38 +-
...st_streaming_mysql_job_schema_change_ignore.out | 20 +
...test_streaming_oceanbase_job_non_column_ddl.out | 52 +
.../test_streaming_oceanbase_job_schema_change.out | 15 +
.../cdc/test_streaming_postgres_job_col_filter.out | 9 +-
.../test_streaming_postgres_job_non_column_ddl.out | 47 +
.../cdc/test_streaming_postgres_job_sc.out | 27 +-
.../test_streaming_postgres_job_sc_advanced.out | 12 +-
...streaming_postgres_job_schema_change_ignore.out | 29 +
.../test_streaming_insert_job_s3_event_aksk.out | 20 +
...est_streaming_insert_job_s3_event_iam_role.out} | 12 +-
...st_streaming_insert_job_s3_event_restart_fe.out | 14 +
.../test_aggregate_all_functions.out | 2 +-
.../count_rollup_empty_input.out | 29 +
.../test_aggregate_all_functions.out | 2 +-
.../test_quantile_state_explicit_boundary.out | 22 +
regression-test/framework/pom.xml | 4 +
.../test_compute_group_name_reconciliation.groovy | 169 +++
.../cdc/test_streaming_mysql_job_col_filter.groovy | 17 +
.../test_streaming_mysql_job_create_alter.groovy | 76 +-
.../test_streaming_mysql_job_non_column_ddl.groovy | 125 +++
.../test_streaming_mysql_job_sc_advanced.groovy | 62 +-
...streaming_mysql_job_schema_change_ignore.groovy | 95 ++
.../cdc/test_streaming_mysql_job_server_id.groovy | 27 +
...t_streaming_oceanbase_job_non_column_ddl.groovy | 125 +++
...st_streaming_oceanbase_job_schema_change.groovy | 45 +
.../test_streaming_postgres_job_col_filter.groovy | 57 +
..._postgres_job_manual_modify_pause_resume.groovy | 26 +-
...st_streaming_postgres_job_non_column_ddl.groovy | 126 +++
.../cdc/test_streaming_postgres_job_sc.groovy | 83 +-
.../test_streaming_postgres_job_sc_advanced.groovy | 53 +-
...eaming_postgres_job_schema_change_ignore.groovy | 123 ++
.../tvf/test_streaming_job_cdc_stream_mysql.groovy | 105 +-
.../test_streaming_insert_job_s3_event_aksk.groovy | 217 ++++
...t_streaming_insert_job_s3_event_iam_role.groovy | 128 +++
...streaming_insert_job_s3_event_restart_fe.groovy | 152 +++
.../test_streaming_insert_job_s3_once.groovy | 6 +-
.../count_rollup_empty_input.groovy | 92 ++
.../test_quantile_state_explicit_boundary.groovy | 70 ++
144 files changed, 8401 insertions(+), 734 deletions(-)
copy fe/fe-core/src/main/java/org/apache/doris/job/offset/s3/{S3Offset.java =>
S3EventOffset.java} (51%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/job/offset/s3/S3EventSourceOffsetProvider.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudClusterCheckerTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTaskParallelismTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/job/offset/s3/S3EventOffsetTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/job/offset/s3/S3EventSourceOffsetProviderTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadJobPersistenceTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/physical/PhysicalPlanVisitorDispatchTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java
copy
regression-test/data/{job_p0/streaming_job/cdc/test_streaming_mysql_job_col_filter.out
=> cloud_p0/node_mgr/test_compute_group_name_reconciliation.out} (52%)
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_non_column_ddl.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_schema_change_ignore.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_non_column_ddl.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_postgres_job_non_column_ddl.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_postgres_job_schema_change_ignore.out
create mode 100644
regression-test/data/job_p0/streaming_job/test_streaming_insert_job_s3_event_aksk.out
copy
regression-test/data/job_p0/streaming_job/{cdc/test_streaming_mysql_job_col_filter.out
=> test_streaming_insert_job_s3_event_iam_role.out} (57%)
create mode 100644
regression-test/data/job_p0/streaming_job/test_streaming_insert_job_s3_event_restart_fe.out
create mode 100644
regression-test/data/nereids_rules_p0/mv/count_rollup_empty_input/count_rollup_empty_input.out
create mode 100644
regression-test/data/query_p0/sql_functions/quantile_state_functions/test_quantile_state_explicit_boundary.out
create mode 100644
regression-test/suites/cloud_p0/node_mgr/test_compute_group_name_reconciliation.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_non_column_ddl.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_schema_change_ignore.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_oceanbase_job_non_column_ddl.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_non_column_ddl.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_postgres_job_schema_change_ignore.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_s3_event_aksk.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_s3_event_iam_role.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/test_streaming_insert_job_s3_event_restart_fe.groovy
create mode 100644
regression-test/suites/nereids_rules_p0/mv/count_rollup_empty_input/count_rollup_empty_input.groovy
create mode 100644
regression-test/suites/query_p0/sql_functions/quantile_state_functions/test_quantile_state_explicit_boundary.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
