Hongshun Wang created FLINK-39212:
-------------------------------------
Summary: CDC yaml will wrong if metadata and physical columns
share same column name
Key: FLINK-39212
URL: https://issues.apache.org/jira/browse/FLINK-39212
Project: Flink
Issue Type: Improvement
Components: Flink CDC
Affects Versions: cdc-3.5.0
Reporter: Hongshun Wang
Fix For: cdc-3.6.0
{code:java}
2025-07-08 17:49:59,128 INFO
org.apache.flink.runtime.executiongraph.ExecutionGraph [] -
Transform:Data -> Partitioning (1/1)
(0630ee078ca577034753901c9b67237e_90bea66de1c231edf33913ecd54406c1_0_0)
switched from RUNNING to FAILED on
job-89d8e079-ceb3-42f3-bc7e-7a435a551eed-taskmanager-1-1 @ 21.224.40.13
(dataPort=43949).org.apache.flink.cdc.runtime.operators.transform.exceptions.TransformException:
Failed to post-transform with CreateTableEvent{tableId=exchange.del_offset,
schema=columns={`id` BIGINT NOT NULL,`biz_type` STRING,`db_shard`
STRING,`table_shard` STRING,`offset` STRING,`update_time` TIMESTAMP(6)},
primaryKeys=id, partitionKeys=pt;blog_db, options=()}for table
exchange.del_offsetfrom schema (Unknown)to schema (Unknown). at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.processElement(PostTransformOperator.java:147)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.streaming.runtime.tasks.OneInputStreamTask$StreamTaskNetworkOutput.emitRecord(OneInputStreamTask.java:246)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.streaming.runtime.io.AbstractStreamTaskNetworkInput.processElement(AbstractStreamTaskNetworkInput.java:217)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.streaming.runtime.io.AbstractStreamTaskNetworkInput.emitNext(AbstractStreamTaskNetworkInput.java:169)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.streaming.runtime.io.StreamOneInputProcessor.processInput(StreamOneInputProcessor.java:68)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.streaming.runtime.tasks.StreamTask.processInput(StreamTask.java:706)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:231)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:1202)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:1146)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.runtime.taskmanager.Task.runWithSystemExitMonitoring(Task.java:976)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.runtime.taskmanager.Task.restoreAndInvoke(Task.java:955)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:768)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.runtime.taskmanager.Task.run(Task.java:580)
~[flink-dist-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
java.lang.Thread.run(Thread.java:991) ~[?:?]Caused by:
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.runtime.CalciteContextException:
At line 0, column 0: Column 'offset' is ambiguous at
jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
~[?:?] at
jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
~[?:?] at
jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:490)
~[?:?] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.runtime.Resources$ExInstWithCause.ex(Resources.java:505)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.SqlUtil.newContextException(SqlUtil.java:932)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.SqlUtil.newContextException(SqlUtil.java:917)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.newValidationError(SqlValidatorImpl.java:5362)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.DelegatingScope.fullyQualify(DelegatingScope.java:292)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl$Expander.visit(SqlValidatorImpl.java:6511)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl$SelectExpander.visit(SqlValidatorImpl.java:6676)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl$SelectExpander.visit(SqlValidatorImpl.java:6662)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.SqlIdentifier.accept(SqlIdentifier.java:324)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl$Expander.go(SqlValidatorImpl.java:6500)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.expandSelectExpr(SqlValidatorImpl.java:6089)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.expandSelectItem(SqlValidatorImpl.java:435)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelectList(SqlValidatorImpl.java:4505)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelect(SqlValidatorImpl.java:3710)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SelectNamespace.validateImpl(SelectNamespace.java:64)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:89)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:1107)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:1078)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.SqlSelect.validate(SqlSelect.java:248)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validateScopedExpression(SqlValidatorImpl.java:1053)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validate(SqlValidatorImpl.java:759)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.runtime.parser.TransformParser.sqlToRel(TransformParser.java:174)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.runtime.parser.TransformParser.generateProjectionColumns(TransformParser.java:288)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.transformSchema(PostTransformOperator.java:395)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.lambda$processCreateTableEvent$0(PostTransformOperator.java:237)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
~[?:?] at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1655)
~[?:?] at
java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) ~[?:?] at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
~[?:?] at
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913)
~[?:?] at
java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:?] at
java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578) ~[?:?]
at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.processCreateTableEvent(PostTransformOperator.java:238)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.processElementInternal(PostTransformOperator.java:189)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.processElement(PostTransformOperator.java:131)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
... 13 moreCaused by:
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorException:
Column 'offset' is ambiguous at
jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
~[?:?] at
jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
~[?:?] at
jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:490)
~[?:?] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.runtime.Resources$ExInstWithCause.ex(Resources.java:505)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.runtime.Resources$ExInst.ex(Resources.java:599)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.SqlUtil.newContextException(SqlUtil.java:932)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.SqlUtil.newContextException(SqlUtil.java:917)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.newValidationError(SqlValidatorImpl.java:5362)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.DelegatingScope.fullyQualify(DelegatingScope.java:292)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl$Expander.visit(SqlValidatorImpl.java:6511)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl$SelectExpander.visit(SqlValidatorImpl.java:6676)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl$SelectExpander.visit(SqlValidatorImpl.java:6662)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.SqlIdentifier.accept(SqlIdentifier.java:324)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl$Expander.go(SqlValidatorImpl.java:6500)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.expandSelectExpr(SqlValidatorImpl.java:6089)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.expandSelectItem(SqlValidatorImpl.java:435)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelectList(SqlValidatorImpl.java:4505)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validateSelect(SqlValidatorImpl.java:3710)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SelectNamespace.validateImpl(SelectNamespace.java:64)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.AbstractNamespace.validate(AbstractNamespace.java:89)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validateNamespace(SqlValidatorImpl.java:1107)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validateQuery(SqlValidatorImpl.java:1078)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.SqlSelect.validate(SqlSelect.java:248)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validateScopedExpression(SqlValidatorImpl.java:1053)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.calcite.shaded.org.apache.calcite.sql.validate.SqlValidatorImpl.validate(SqlValidatorImpl.java:759)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11] at
org.apache.flink.cdc.runtime.parser.TransformParser.sqlToRel(TransformParser.java:174)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.runtime.parser.TransformParser.generateProjectionColumns(TransformParser.java:288)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.transformSchema(PostTransformOperator.java:395)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.lambda$processCreateTableEvent$0(PostTransformOperator.java:237)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:195)
~[?:?] at
java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1655)
~[?:?] at
java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) ~[?:?] at
java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
~[?:?] at
java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913)
~[?:?] at
java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) ~[?:?] at
java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578) ~[?:?]
at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.processCreateTableEvent(PostTransformOperator.java:238)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.processElementInternal(PostTransformOperator.java:189)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
at
org.apache.flink.cdc.runtime.operators.transform.PostTransformOperator.processElement(PostTransformOperator.java:131)
~[flink-cdc-cli-bundle-1.20-vvr-11.1.2-jdk11.jar:1.20-vvr-11.1.2-jdk11]
... 13 more {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
