[
https://issues.apache.org/jira/browse/FLINK-38921?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=18052010#comment-18052010
]
谢波 commented on FLINK-38921:
----------------------------
osc feat
https://github.com/apache/flink-cdc/pull/3668/files#diff-4165905722f8cfa19dc970472d49a9e64fde441a4a8763d2bc35ba64df4787dc
> osc support custom regx
> -----------------------
>
> Key: FLINK-38921
> URL: https://issues.apache.org/jira/browse/FLINK-38921
> Project: Flink
> Issue Type: New Feature
> Components: Flink CDC
> Environment: flink-cdc 3.5.0
> Reporter: 谢波
> Priority: Major
>
> aliyun rds mysql non lock ddl change binlog like this
> {"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448976159,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_ogl_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850094","file":"mysql-bin.105702","pos":210384716,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210384716,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448976,\"file\":\"mysql-bin.105702\",\"pos\":210385137,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850093\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"CREATE
> TABLE `adv`.`tp_10253221_ogl_test_cdc` (\\n\\tid BIGINT
> auto_increment,\\n\\tlast_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
> ON UPDATE CURRENT_TIMESTAMP,\\n\\t`key` VARCHAR (64) charset utf8 NOT
> NULL,\\n\\t`value` VARCHAR (4096) charset utf8 NOT NULL,\\n\\tPRIMARY KEY
> (id),\\n\\tUNIQUE KEY key_uidx (`key`)\\n) auto_increment =
> 512\",\"tableChanges\":[\{\"type\":\"CREATE\",\"id\":\"\\\"adv\\\".\\\"tp_10253221_ogl_test_cdc\\\"\",\"table\":{\"defaultCharsetName\":null,\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"last_update\",\"jdbcType\":2014,\"typeName\":\"TIMESTAMP\",\"typeExpression\":\"TIMESTAMP\",\"charsetName\":null,\"position\":2,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"defaultValueExpression\":\"1970-01-01
>
> 00:00:00\",\"enumValues\":[]},\{\"name\":\"key\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":64,\"position\":3,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"value\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":4096,\"position\":4,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]}]},\"comment\":null}]}"}
> {"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448976326,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_ogt_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850096","file":"mysql-bin.105702","pos":210385502,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210385502,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448976,\"file\":\"mysql-bin.105702\",\"pos\":210385661,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850095\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"CREATE
> TABLE `adv`.`tp_10253221_ogt_test_cdc` LIKE
> `adv`.`test_cdc`\",\"tableChanges\":[\{\"type\":\"CREATE\",\"id\":\"\\\"adv\\\".\\\"tp_10253221_ogt_test_cdc\\\"\",\"table\":{\"defaultCharsetName\":\"utf8\",\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"length\":20,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"name\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":100,\"position\":2,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"pt\",\"jdbcType\":2014,\"typeName\":\"TIMESTAMP\",\"typeExpression\":\"TIMESTAMP\",\"charsetName\":null,\"length\":3,\"position\":3,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"defaultValueExpression\":\"1970-01-01
>
> 00:00:00\",\"enumValues\":[]},\{\"name\":\"f1\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":4,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f2\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":5,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]}]},\"comment\":null}]}"}
> {"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448976505,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_ogt_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850098","file":"mysql-bin.105702","pos":210386098,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210386098,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448976,\"file\":\"mysql-bin.105702\",\"pos\":210386242,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850097\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"ALTER
> TABLE `adv`.`tp_10253221_ogt_test_cdc`\\n\\tADD COLUMN f3
> varchar(10)\",\"tableChanges\":[\{\"type\":\"ALTER\",\"id\":\"\\\"adv\\\".\\\"tp_10253221_ogt_test_cdc\\\"\",\"table\":{\"defaultCharsetName\":\"utf8\",\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"length\":20,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"name\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":100,\"position\":2,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"pt\",\"jdbcType\":2014,\"typeName\":\"TIMESTAMP\",\"typeExpression\":\"TIMESTAMP\",\"charsetName\":null,\"length\":3,\"position\":3,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"defaultValueExpression\":\"1970-01-01
>
> 00:00:00\",\"enumValues\":[]},\{\"name\":\"f1\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":4,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f2\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":5,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f3\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":6,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]}]},\"comment\":null}]}"}
> {"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448977944,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_del_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850104","file":"mysql-bin.105702","pos":210387791,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210387791,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448977,\"file\":\"mysql-bin.105702\",\"pos\":210387975,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850103\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"CREATE
> TABLE `adv`.`tp_10253221_del_test_cdc` (`id` bigint AUTO_INCREMENT primary
> key)\",\"tableChanges\":[\{\"type\":\"CREATE\",\"id\":\"\\\"adv\\\".\\\"tp_10253221_del_test_cdc\\\"\",\"table\":{\"defaultCharsetName\":null,\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]}]},\"comment\":null}]}"}
> {"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448978441,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_del_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850108","file":"mysql-bin.105702","pos":210389118,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210389118,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448978,\"file\":\"mysql-bin.105702\",\"pos\":210389266,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850107\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"DROP
> TABLE IF EXISTS `adv`.`tp_10253221_del_test_cdc`\",\"tableChanges\":[]}"}
> {"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448978455,"snapshot":"false","db":"adv","sequence":null,"table":"test_cdc,tp_10253221_del_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850109","file":"mysql-bin.105702","pos":210389314,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210389314,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448978,\"file\":\"mysql-bin.105702\",\"pos\":210389649,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850108\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"RENAME
> TABLE `adv`.`test_cdc` to
> `adv`.`tp_10253221_del_test_cdc`\",\"tableChanges\":[\{\"type\":\"ALTER\",\"id\":\"\\\"adv\\\".\\\"tp_10253221_del_test_cdc\\\"\",\"previousId\":\"\\\"adv\\\".\\\"test_cdc\\\"\",\"table\":{\"defaultCharsetName\":\"utf8\",\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"length\":20,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"name\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":100,\"position\":2,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"pt\",\"jdbcType\":2014,\"typeName\":\"TIMESTAMP\",\"typeExpression\":\"TIMESTAMP\",\"charsetName\":null,\"length\":3,\"position\":3,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"defaultValueExpression\":\"1970-01-01
>
> 00:00:00\",\"enumValues\":[]},\{\"name\":\"f1\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":4,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f2\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":5,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]}]},\"comment\":null}]}"}
> {"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448978455,"snapshot":"false","db":"adv","sequence":null,"table":"test_cdc,tp_10253221_del_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850109","file":"mysql-bin.105702","pos":210389314,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210389314,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448978,\"file\":\"mysql-bin.105702\",\"pos\":210389649,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850108\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"RENAME
> TABLE `adv`.`tp_10253221_ogt_test_cdc` to
> `adv`.`test_cdc`\",\"tableChanges\":[\{\"type\":\"ALTER\",\"id\":\"\\\"adv\\\".\\\"test_cdc\\\"\",\"previousId\":\"\\\"adv\\\".\\\"tp_10253221_ogt_test_cdc\\\"\",\"table\":{\"defaultCharsetName\":\"utf8\",\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"length\":20,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"name\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":100,\"position\":2,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"pt\",\"jdbcType\":2014,\"typeName\":\"TIMESTAMP\",\"typeExpression\":\"TIMESTAMP\",\"charsetName\":null,\"length\":3,\"position\":3,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"defaultValueExpression\":\"1970-01-01
>
> 00:00:00\",\"enumValues\":[]},\{\"name\":\"f1\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":4,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f2\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":5,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f3\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":6,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]}]},\"comment\":null}]}"}
> {"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448978729,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_ogl_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850110","file":"mysql-bin.105702","pos":210389697,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210389697,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448978,\"file\":\"mysql-bin.105702\",\"pos\":210389835,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850109\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"DROP
> TABLE `adv`.`tp_10253221_ogl_test_cdc`\",\"tableChanges\":[]}"}
> {"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448979181,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_del_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850111","file":"mysql-bin.105702","pos":210389883,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210389883,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448979,\"file\":\"mysql-bin.105702\",\"pos\":210390021,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850110\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"DROP
> TABLE `adv`.`tp_10253221_del_test_cdc`\",\"tableChanges\":[]}"}
> but flink cdc osc can't support the ddl like this.
> we need custom osc regx to match the table name (like
> tp_10253221_del_test_cdc).
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
