谢波 created FLINK-38921:
--------------------------
Summary: osc support custom regx
Key: FLINK-38921
URL: https://issues.apache.org/jira/browse/FLINK-38921
Project: Flink
Issue Type: New Feature
Components: Flink CDC
Environment: flink-cdc 3.5.0
Reporter: 谢波
aliyun rds mysql non lock ddl change binlog like this
{"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448976159,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_ogl_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850094","file":"mysql-bin.105702","pos":210384716,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210384716,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448976,\"file\":\"mysql-bin.105702\",\"pos\":210385137,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850093\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"CREATE
TABLE `adv`.`tp_10253221_ogl_test_cdc` (\\n\\tid BIGINT
auto_increment,\\n\\tlast_update TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
ON UPDATE CURRENT_TIMESTAMP,\\n\\t`key` VARCHAR (64) charset utf8 NOT
NULL,\\n\\t`value` VARCHAR (4096) charset utf8 NOT NULL,\\n\\tPRIMARY KEY
(id),\\n\\tUNIQUE KEY key_uidx (`key`)\\n) auto_increment =
512\",\"tableChanges\":[\{\"type\":\"CREATE\",\"id\":\"\\\"adv\\\".\\\"tp_10253221_ogl_test_cdc\\\"\",\"table\":{\"defaultCharsetName\":null,\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"last_update\",\"jdbcType\":2014,\"typeName\":\"TIMESTAMP\",\"typeExpression\":\"TIMESTAMP\",\"charsetName\":null,\"position\":2,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"defaultValueExpression\":\"1970-01-01
00:00:00\",\"enumValues\":[]},\{\"name\":\"key\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":64,\"position\":3,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"value\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":4096,\"position\":4,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]}]},\"comment\":null}]}"}
{"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448976326,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_ogt_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850096","file":"mysql-bin.105702","pos":210385502,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210385502,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448976,\"file\":\"mysql-bin.105702\",\"pos\":210385661,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850095\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"CREATE
TABLE `adv`.`tp_10253221_ogt_test_cdc` LIKE
`adv`.`test_cdc`\",\"tableChanges\":[\{\"type\":\"CREATE\",\"id\":\"\\\"adv\\\".\\\"tp_10253221_ogt_test_cdc\\\"\",\"table\":{\"defaultCharsetName\":\"utf8\",\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"length\":20,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"name\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":100,\"position\":2,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"pt\",\"jdbcType\":2014,\"typeName\":\"TIMESTAMP\",\"typeExpression\":\"TIMESTAMP\",\"charsetName\":null,\"length\":3,\"position\":3,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"defaultValueExpression\":\"1970-01-01
00:00:00\",\"enumValues\":[]},\{\"name\":\"f1\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":4,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f2\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":5,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]}]},\"comment\":null}]}"}
{"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448976505,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_ogt_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850098","file":"mysql-bin.105702","pos":210386098,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210386098,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448976,\"file\":\"mysql-bin.105702\",\"pos\":210386242,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850097\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"ALTER
TABLE `adv`.`tp_10253221_ogt_test_cdc`\\n\\tADD COLUMN f3
varchar(10)\",\"tableChanges\":[\{\"type\":\"ALTER\",\"id\":\"\\\"adv\\\".\\\"tp_10253221_ogt_test_cdc\\\"\",\"table\":{\"defaultCharsetName\":\"utf8\",\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"length\":20,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"name\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":100,\"position\":2,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"pt\",\"jdbcType\":2014,\"typeName\":\"TIMESTAMP\",\"typeExpression\":\"TIMESTAMP\",\"charsetName\":null,\"length\":3,\"position\":3,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"defaultValueExpression\":\"1970-01-01
00:00:00\",\"enumValues\":[]},\{\"name\":\"f1\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":4,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f2\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":5,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f3\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":6,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]}]},\"comment\":null}]}"}
{"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448977944,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_del_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850104","file":"mysql-bin.105702","pos":210387791,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210387791,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448977,\"file\":\"mysql-bin.105702\",\"pos\":210387975,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850103\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"CREATE
TABLE `adv`.`tp_10253221_del_test_cdc` (`id` bigint AUTO_INCREMENT primary
key)\",\"tableChanges\":[\{\"type\":\"CREATE\",\"id\":\"\\\"adv\\\".\\\"tp_10253221_del_test_cdc\\\"\",\"table\":{\"defaultCharsetName\":null,\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]}]},\"comment\":null}]}"}
{"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448978441,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_del_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850108","file":"mysql-bin.105702","pos":210389118,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210389118,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448978,\"file\":\"mysql-bin.105702\",\"pos\":210389266,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850107\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"DROP
TABLE IF EXISTS `adv`.`tp_10253221_del_test_cdc`\",\"tableChanges\":[]}"}
{"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448978455,"snapshot":"false","db":"adv","sequence":null,"table":"test_cdc,tp_10253221_del_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850109","file":"mysql-bin.105702","pos":210389314,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210389314,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448978,\"file\":\"mysql-bin.105702\",\"pos\":210389649,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850108\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"RENAME
TABLE `adv`.`test_cdc` to
`adv`.`tp_10253221_del_test_cdc`\",\"tableChanges\":[\{\"type\":\"ALTER\",\"id\":\"\\\"adv\\\".\\\"tp_10253221_del_test_cdc\\\"\",\"previousId\":\"\\\"adv\\\".\\\"test_cdc\\\"\",\"table\":{\"defaultCharsetName\":\"utf8\",\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"length\":20,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"name\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":100,\"position\":2,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"pt\",\"jdbcType\":2014,\"typeName\":\"TIMESTAMP\",\"typeExpression\":\"TIMESTAMP\",\"charsetName\":null,\"length\":3,\"position\":3,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"defaultValueExpression\":\"1970-01-01
00:00:00\",\"enumValues\":[]},\{\"name\":\"f1\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":4,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f2\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":5,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]}]},\"comment\":null}]}"}
{"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448978455,"snapshot":"false","db":"adv","sequence":null,"table":"test_cdc,tp_10253221_del_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850109","file":"mysql-bin.105702","pos":210389314,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210389314,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448978,\"file\":\"mysql-bin.105702\",\"pos\":210389649,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850108\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"RENAME
TABLE `adv`.`tp_10253221_ogt_test_cdc` to
`adv`.`test_cdc`\",\"tableChanges\":[\{\"type\":\"ALTER\",\"id\":\"\\\"adv\\\".\\\"test_cdc\\\"\",\"previousId\":\"\\\"adv\\\".\\\"tp_10253221_ogt_test_cdc\\\"\",\"table\":{\"defaultCharsetName\":\"utf8\",\"primaryKeyColumnNames\":[\"id\"],\"columns\":[{\"name\":\"id\",\"jdbcType\":-5,\"typeName\":\"BIGINT\",\"typeExpression\":\"BIGINT\",\"charsetName\":null,\"length\":20,\"position\":1,\"optional\":false,\"autoIncremented\":true,\"generated\":true,\"comment\":null,\"hasDefaultValue\":false,\"enumValues\":[]},\{\"name\":\"name\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":100,\"position\":2,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"pt\",\"jdbcType\":2014,\"typeName\":\"TIMESTAMP\",\"typeExpression\":\"TIMESTAMP\",\"charsetName\":null,\"length\":3,\"position\":3,\"optional\":false,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"defaultValueExpression\":\"1970-01-01
00:00:00\",\"enumValues\":[]},\{\"name\":\"f1\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":4,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f2\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":5,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]},\{\"name\":\"f3\",\"jdbcType\":12,\"typeName\":\"VARCHAR\",\"typeExpression\":\"VARCHAR\",\"charsetName\":\"utf8\",\"length\":10,\"position\":6,\"optional\":true,\"autoIncremented\":false,\"generated\":false,\"comment\":null,\"hasDefaultValue\":true,\"enumValues\":[]}]},\"comment\":null}]}"}
{"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448978729,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_ogl_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850110","file":"mysql-bin.105702","pos":210389697,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210389697,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448978,\"file\":\"mysql-bin.105702\",\"pos\":210389835,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850109\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"DROP
TABLE `adv`.`tp_10253221_ogl_test_cdc`\",\"tableChanges\":[]}"}
{"source":\{"version":"1.9.8.Final","connector":"mysql","name":"mysql_binlog_source","ts_ms":1768448979181,"snapshot":"false","db":"adv","sequence":null,"table":"tp_10253221_del_test_cdc","server_id":650127,"gtid":"368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:2898850111","file":"mysql-bin.105702","pos":210389883,"row":0,"thread":null,"query":null},"historyRecord":"\{\"source\":{\"file\":\"mysql-bin.105702\",\"pos\":210389883,\"server_id\":650127},\"position\":\{\"transaction_id\":null,\"ts_sec\":1768448979,\"file\":\"mysql-bin.105702\",\"pos\":210390021,\"gtids\":\"16ec322a-29ef-11ee-8ef7-b8599f2795e8:1-25226030270,368dd6ab-29ef-11ee-8ef8-b8599f1efa4f:1-2898850110\",\"server_id\":650127},\"databaseName\":\"adv\",\"ddl\":\"DROP
TABLE `adv`.`tp_10253221_del_test_cdc`\",\"tableChanges\":[]}"}
but flink cdc osc can't support the ddl like this.
we need custom osc regx to match the table name (like tp_10253221_del_test_cdc).
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
