This is an automated email from the ASF dual-hosted git repository.
leonard pushed a change to branch release-3.2
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
from ab64eb3b5 [FLINK-36056][cdc][connector/elasticsearch] Change
flink.connector.elasticsearch.version to a released version (#3543)
new 74f58cd31 [FLINK-36082][pipeline-connector][kafka] Fix lamda
NotSerializableException in KafkaDataSink
new 568888af7 [FLINK-36088][pipeline-connector][paimon] Fix NPE in
BucketAssignOperator when job restoration
new 9f358ab1d [FLINK-36076][minor][cdc-runtime] Set isSchemaChangeApplying
as volatile for thread safe consideration
new 39afc4fc8 [FLINK-36081][cdc-connector][mysql] Remove the schemas of
outdated tables in the BinlogSplit when restart (#3548)
new 5a8075744 [FLINK-36094][cdc-runtime] Improve the Exception that
SchemaRegistryRequestHandler thrown
new 6dd1c11fe [FLINK-36114][cdc-runtime] Make SchemaRegistryRequestHandler
thread safe by blocking subsequent schemaChangeEvent
new dacbe7c34 [FLINK-36092][cdc-runtime] Fix schema evolution failure with
wildcarded transform rule
new 803d43865 [hotfix][cdc-runtime] Fix schema registry hanging in
multiple parallelism
new daf28ddc9 [FLINK-35056][cdc-connector/sqlserver] Fix scale mapping
from SQL Server TIMESTAMP to Flink SQL timestamp
new debd43cdd [FLINK-36128][cdc-runtime] Fix potential unrecoverable
in-flight data exception by promoting LENIENT as the default schema change
behavior
new 9816848c4 [hotfix][cdc-runtime] Keep upstream pending requests in
order to avoid checkpoint hanging
new 1ebefd6d2 [hotfix] Run schema coordinator logic asynchronously to
avoid blocking the main thread
The 12 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../cli/parser/YamlPipelineDefinitionParser.java | 38 +-
.../parser/YamlPipelineDefinitionParserTest.java | 36 +-
.../cdc/common/event/SchemaChangeEventType.java | 20 +-
.../common/exceptions/SchemaEvolveException.java | 6 +-
.../UnsupportedSchemaChangeEventException.java | 28 +-
.../flink/cdc/common/pipeline/PipelineOptions.java | 2 +-
.../flink/cdc/common/utils/ChangeEventUtils.java | 8 +-
.../apache/flink/cdc/common/utils/SchemaUtils.java | 71 ++
.../cdc/common/utils/ChangeEventUtilsTest.java | 67 +-
.../cdc/composer/flink/FlinkPipelineComposer.java | 3 +-
.../flink/translator/DataSourceTranslator.java | 7 +-
.../flink/FlinkPipelineComposerITCase.java | 29 +-
.../flink/FlinkPipelineTransformITCase.java | 635 +++++++++++++++++
.../cdc/composer/flink/FlinkPipelineUdfITCase.java | 41 +-
.../cdc/connectors/kafka/sink/KafkaDataSink.java | 35 +-
.../kafka/sink/KafkaMetaDataApplier.java | 36 +-
.../source/reader/MySqlPipelineRecordEmitter.java | 62 +-
.../mysql/source/MySqlFullTypesITCase.java | 26 +-
.../mysql/source/MySqlPipelineITCase.java | 30 +-
.../mysql/testutils/MySqSourceTestUtils.java | 19 +
.../sink/v2/bucket/BucketAssignOperator.java | 2 +-
.../cdc/connectors/values/ValuesDatabase.java | 3 +-
.../mysql/source/split/MySqlBinlogSplit.java | 31 +-
.../mysql/source/split/MySqlBinlogSplitTest.java | 140 ++++
.../sqlserver/source/utils/SqlServerTypeUtils.java | 4 +-
.../read/fetch/SqlServerScanFetchTaskTest.java | 26 +
.../src/test/resources/ddl/pk.sql} | 19 +-
.../cdc/pipeline/tests/MySqlToDorisE2eITCase.java | 10 +-
.../flink/cdc/pipeline/tests/MysqlE2eITCase.java | 243 ++++---
.../flink/cdc/pipeline/tests/RouteE2eITCase.java | 37 +-
.../cdc/pipeline/tests/SchemaEvolveE2eITCase.java | 34 +-
....java => SchemaEvolvingTransformE2eITCase.java} | 98 +--
.../cdc/pipeline/tests/TransformE2eITCase.java | 388 +++++++++-
.../flink/cdc/pipeline/tests/UdfE2eITCase.java | 8 +-
.../tests/utils/PipelineTestEnvironment.java | 2 +
.../tests/SchemaRegistryMigrationMock.java | 2 +
.../runtime/operators/schema/SchemaOperator.java | 145 ++--
.../schema/coordinator/SchemaManager.java | 84 +++
.../schema/coordinator/SchemaRegistry.java | 270 ++++---
.../schema/coordinator/SchemaRegistryProvider.java | 54 +-
.../coordinator/SchemaRegistryRequestHandler.java | 424 ++++++-----
.../event/ApplyEvolvedSchemaChangeRequest.java | 73 --
.../event/ApplyEvolvedSchemaChangeResponse.java | 32 -
.../event/ApplyOriginalSchemaChangeRequest.java | 71 --
.../event/ApplyOriginalSchemaChangeResponse.java | 31 -
.../schema/event/RefreshPendingListsRequest.java | 27 -
.../schema/event/RefreshPendingListsResponse.java | 26 -
.../schema/event/ReleaseUpstreamRequest.java | 32 -
.../event/SchemaChangeProcessingResponse.java | 4 +-
.../schema/event/SchemaChangeRequest.java | 15 +-
.../schema/event/SchemaChangeResponse.java | 75 +-
...sponse.java => SchemaChangeResultResponse.java} | 48 +-
.../operators/transform/PostTransformOperator.java | 55 +-
.../operators/transform/PreTransformOperator.java | 101 ++-
.../flink/cdc/runtime/parser/TransformParser.java | 22 +
.../runtime/partitioning/PrePartitionOperator.java | 13 +-
.../runtime/operators/schema/SchemaEvolveTest.java | 14 +-
.../transform/PreTransformOperatorTest.java | 11 +-
.../TransformOperatorWithSchemaEvolveTest.java | 792 +++++++++++++++++++++
.../operators/EventOperatorTestHarness.java | 44 +-
.../MockedOperatorCoordinatorContext.java | 29 +-
.../schema/CollectingMetadataApplier.java | 4 +-
62 files changed, 3575 insertions(+), 1167 deletions(-)
copy
flink-cdc-common/src/main/java/org/apache/flink/cdc/common/sink/MetadataApplier.java
=>
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka/src/main/java/org/apache/flink/cdc/connectors/kafka/sink/KafkaMetaDataApplier.java
(56%)
create mode 100644
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/split/MySqlBinlogSplitTest.java
copy
flink-cdc-connect/flink-cdc-source-connectors/{flink-connector-vitess-cdc/src/test/resources/ddl/inventory.sql
=> flink-connector-sqlserver-cdc/src/test/resources/ddl/pk.sql} (76%)
copy
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/{SchemaEvolveE2eITCase.java
=> SchemaEvolvingTransformE2eITCase.java} (75%)
delete mode 100644
flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/ApplyEvolvedSchemaChangeRequest.java
delete mode 100644
flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/ApplyEvolvedSchemaChangeResponse.java
delete mode 100644
flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/ApplyOriginalSchemaChangeRequest.java
delete mode 100644
flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/ApplyOriginalSchemaChangeResponse.java
delete mode 100644
flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/RefreshPendingListsRequest.java
delete mode 100644
flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/RefreshPendingListsResponse.java
delete mode 100644
flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/ReleaseUpstreamRequest.java
rename
flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/schema/event/{ReleaseUpstreamResponse.java
=> SchemaChangeResultResponse.java} (53%)
create mode 100644
flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/transform/TransformOperatorWithSchemaEvolveTest.java
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc/src/main/java/org/apache/flink/cdc/connectors/oracle/source/reader/fetch/StoppableChangeEventSourceContext.java
=>
flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/testutils/operators/MockedOperatorCoordinatorContext.java
(52%)
