This is an automated email from the ASF dual-hosted git repository.
ruanhang1993 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink-cdc.git
from e0d6d1d1a [FLINK-35442][cdc-connect][kafka] add key.format and
partition.strategy option to make sure the same record sending to the same
partition. (#3522)
add e2bb917ad [FLINK-34876][transform] Support UDF functions in transform
(#3465)
No new revisions were added by this update.
Summary of changes:
README.md | 75 +-
docs/content.zh/docs/core-concept/data-pipeline.md | 17 +-
docs/content.zh/docs/core-concept/transform.md | 69 ++
docs/content/docs/core-concept/data-pipeline.md | 18 +-
docs/content/docs/core-concept/transform.md | 69 ++
.../cli/parser/YamlPipelineDefinitionParser.java | 31 +-
.../parser/YamlPipelineDefinitionParserTest.java | 40 +
.../definitions/pipeline-definition-with-udf.yaml | 21 +-
.../UserDefinedFunction.java} | 24 +-
flink-cdc-composer/pom.xml | 14 +
.../flink/cdc/composer/definition/PipelineDef.java | 12 +-
.../flink/cdc/composer/definition/UdfDef.java | 39 +-
.../cdc/composer/flink/FlinkPipelineComposer.java | 8 +-
.../flink/translator/TransformTranslator.java | 19 +-
.../flink/FlinkPipelineComposerITCase.java | 35 +-
.../flink/FlinkPipelineTransformITCase.java | 1 +
.../cdc/composer/flink/FlinkPipelineUdfITCase.java | 811 +++++++++++++++++++++
.../flink-cdc-pipeline-e2e-tests/pom.xml | 34 +
.../flink/cdc/pipeline/tests/UdfE2eITCase.java | 403 ++++++++++
flink-cdc-pipeline-udf-examples/pom.xml | 104 +++
.../cdc/udf/examples/java/AddOneFunctionClass.java | 16 +-
.../cdc/udf/examples/java/FormatFunctionClass.java | 12 +-
.../udf/examples/java/LifecycleFunctionClass.java | 27 +-
.../udf/examples/java/TypeHintFunctionClass.java | 28 +-
.../cdc/udf/examples/java/TypeOfFunctionClass.java | 27 +-
.../udf/examples/java/AddOneFunctionClass.java | 19 +-
.../udf/examples/java/FormatFunctionClass.java | 12 +-
.../udf/examples/java/TypeOfFunctionClass.java | 27 +-
.../udf/examples/scala/AddOneFunctionClass.scala | 21 +-
.../udf/examples/scala/FormatFunctionClass.scala | 17 +-
.../examples/scala/LifecycleFunctionClass.scala | 30 +-
.../udf/examples/scala/TypeHintFunctionClass.scala | 28 +-
.../udf/examples/scala/TypeOfFunctionClass.scala | 18 +-
.../udf/examples/scala/AddOneFunctionClass.scala | 21 +-
.../udf/examples/scala/FormatFunctionClass.scala | 21 +-
.../udf/examples/scala/TypeOfFunctionClass.scala | 18 +-
flink-cdc-runtime/pom.xml | 6 +
.../operators/transform/PostTransformOperator.java | 133 +++-
.../operators/transform/PreTransformOperator.java | 33 +-
.../transform/ProjectionColumnProcessor.java | 32 +-
.../transform/TransformExpressionCompiler.java | 19 +-
.../operators/transform/TransformFilter.java | 6 +-
.../transform/TransformFilterProcessor.java | 31 +-
.../transform/TransformProjectionProcessor.java | 44 +-
.../transform/UserDefinedFunctionDescriptor.java | 138 ++++
.../flink/cdc/runtime/parser/JaninoCompiler.java | 92 ++-
.../flink/cdc/runtime/parser/TransformParser.java | 67 +-
.../UserDefinedFunctionDescriptorTest.java | 97 +++
.../cdc/runtime/parser/TransformParserTest.java | 75 +-
pom.xml | 3 +
50 files changed, 2616 insertions(+), 346 deletions(-)
copy
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/test/resources/pipeline-test.yaml
=>
flink-cdc-cli/src/test/resources/definitions/pipeline-definition-with-udf.yaml
(70%)
copy
flink-cdc-common/src/main/java/org/apache/flink/cdc/common/{sink/FlinkSinkProvider.java
=> udf/UserDefinedFunction.java} (61%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/source/assigner/state/StreamPendingSplitsState.java
=>
flink-cdc-composer/src/main/java/org/apache/flink/cdc/composer/definition/UdfDef.java
(56%)
create mode 100644
flink-cdc-composer/src/test/java/org/apache/flink/cdc/composer/flink/FlinkPipelineUdfITCase.java
create mode 100644
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests/src/test/java/org/apache/flink/cdc/pipeline/tests/UdfE2eITCase.java
create mode 100644 flink-cdc-pipeline-udf-examples/pom.xml
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/main/java/org/apache/flink/cdc/connectors/oceanbase/utils/OceanBaseUtils.java
=>
flink-cdc-pipeline-udf-examples/src/main/java/org/apache/flink/cdc/udf/examples/java/AddOneFunctionClass.java
(66%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/main/java/org/apache/flink/cdc/connectors/oceanbase/utils/OceanBaseUtils.java
=>
flink-cdc-pipeline-udf-examples/src/main/java/org/apache/flink/cdc/udf/examples/java/FormatFunctionClass.java
(70%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/test/java/org/apache/flink/cdc/connectors/base/testutils/MySqlVersion.java
=>
flink-cdc-pipeline-udf-examples/src/main/java/org/apache/flink/cdc/udf/examples/java/LifecycleFunctionClass.java
(59%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/event/SchemaDataTypeInference.java
=>
flink-cdc-pipeline-udf-examples/src/main/java/org/apache/flink/cdc/udf/examples/java/TypeHintFunctionClass.java
(60%)
copy
flink-cdc-common/src/main/java/org/apache/flink/cdc/common/utils/EncodingUtils.java
=>
flink-cdc-pipeline-udf-examples/src/main/java/org/apache/flink/cdc/udf/examples/java/TypeOfFunctionClass.java
(59%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/Validator.java
=>
flink-cdc-pipeline-udf-examples/src/main/java/org/apache/flink/udf/examples/java/AddOneFunctionClass.java
(66%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc/src/main/java/org/apache/flink/cdc/connectors/oceanbase/utils/OceanBaseUtils.java
=>
flink-cdc-pipeline-udf-examples/src/main/java/org/apache/flink/udf/examples/java/FormatFunctionClass.java
(71%)
copy
flink-cdc-common/src/main/java/org/apache/flink/cdc/common/utils/EncodingUtils.java
=>
flink-cdc-pipeline-udf-examples/src/main/java/org/apache/flink/udf/examples/java/TypeOfFunctionClass.java
(59%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/table/StartupMode.java
=>
flink-cdc-pipeline-udf-examples/src/main/scala/org/apache/flink/cdc/udf/examples/scala/AddOneFunctionClass.scala
(70%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-debezium/src/main/java/org/apache/flink/cdc/debezium/Validator.java
=>
flink-cdc-pipeline-udf-examples/src/main/scala/org/apache/flink/cdc/udf/examples/scala/FormatFunctionClass.scala
(69%)
copy
flink-cdc-common/src/main/java/org/apache/flink/cdc/common/route/RouteRule.java
=>
flink-cdc-pipeline-udf-examples/src/main/scala/org/apache/flink/cdc/udf/examples/scala/LifecycleFunctionClass.scala
(59%)
copy
flink-cdc-common/src/main/java/org/apache/flink/cdc/common/utils/EncodingUtils.java
=>
flink-cdc-pipeline-udf-examples/src/main/scala/org/apache/flink/cdc/udf/examples/scala/TypeHintFunctionClass.scala
(59%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/relational/handler/SchemaChangeEventHandler.java
=>
flink-cdc-pipeline-udf-examples/src/main/scala/org/apache/flink/cdc/udf/examples/scala/TypeOfFunctionClass.scala
(63%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/table/StartupMode.java
=>
flink-cdc-pipeline-udf-examples/src/main/scala/org/apache/flink/udf/examples/scala/AddOneFunctionClass.scala
(70%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/table/StartupMode.java
=>
flink-cdc-pipeline-udf-examples/src/main/scala/org/apache/flink/udf/examples/scala/FormatFunctionClass.scala
(70%)
copy
flink-cdc-connect/flink-cdc-source-connectors/flink-cdc-base/src/main/java/org/apache/flink/cdc/connectors/base/relational/handler/SchemaChangeEventHandler.java
=>
flink-cdc-pipeline-udf-examples/src/main/scala/org/apache/flink/udf/examples/scala/TypeOfFunctionClass.scala
(64%)
create mode 100644
flink-cdc-runtime/src/main/java/org/apache/flink/cdc/runtime/operators/transform/UserDefinedFunctionDescriptor.java
create mode 100644
flink-cdc-runtime/src/test/java/org/apache/flink/cdc/runtime/operators/transform/UserDefinedFunctionDescriptorTest.java
