Copilot commented on code in PR #4357: URL: https://github.com/apache/flink-cdc/pull/4357#discussion_r3019610534
########## flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/source/connection/ReaderConnectionTest.java: ########## @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.mysql.source.connection; + +import org.apache.flink.cdc.connectors.mysql.debezium.DebeziumUtils; +import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig; +import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfigFactory; +import org.apache.flink.cdc.connectors.mysql.table.StartupOptions; + +import io.debezium.connector.mysql.MySqlConnection; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.time.ZoneId; +import java.util.Properties; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests for reader hostname connection handling. Verifies that the reader connection is used when + * configured, and falls back to the writer connection when not configured. + */ +class ReaderConnectionTest { + + private static final String WRITER_HOSTNAME = "writer-host"; + private static final String READER_HOSTNAME = "reader-host"; + private static final int PORT = 3306; + + @Test + void testReaderHostnameConfigured() { + MySqlSourceConfig config = createConfig(WRITER_HOSTNAME, READER_HOSTNAME); + + assertThat(config.getHostname()).isEqualTo(WRITER_HOSTNAME); + assertThat(config.getSnapshotHostname()).isEqualTo(READER_HOSTNAME); + } + + @Test + void testReaderHostnameNotConfigured() { + MySqlSourceConfig config = createConfig(WRITER_HOSTNAME, null); + + assertThat(config.getHostname()).isEqualTo(WRITER_HOSTNAME); + assertThat(config.getSnapshotHostname()).isNull(); + } + + @Test + void testCreateReaderMySqlConnectionWithReaderHostname() { + MySqlSourceConfig config = createConfig(WRITER_HOSTNAME, READER_HOSTNAME); + + // Create reader connection - should use reader hostname + MySqlConnection readerConnection = DebeziumUtils.createSnapshotMySqlConnection(config); + String readerConnectionString = readerConnection.connectionString(); + + // Create writer connection - should use writer hostname + MySqlConnection writerConnection = DebeziumUtils.createMySqlConnection(config); + String writerConnectionString = writerConnection.connectionString(); + + // Reader connection should contain reader hostname + assertThat(readerConnectionString).contains(READER_HOSTNAME); + assertThat(readerConnectionString).doesNotContain(WRITER_HOSTNAME); + + // Writer connection should contain writer hostname + assertThat(writerConnectionString).contains(WRITER_HOSTNAME); + assertThat(writerConnectionString).doesNotContain(READER_HOSTNAME); + } + + @Test + void testCreateReaderMySqlConnectionFallsBackToWriterWhenNoReaderConfigured() { + MySqlSourceConfig config = createConfig(WRITER_HOSTNAME, null); + + // Create reader connection - should fall back to writer hostname + MySqlConnection readerConnection = DebeziumUtils.createSnapshotMySqlConnection(config); + String readerConnectionString = readerConnection.connectionString(); + + // Create writer connection + MySqlConnection writerConnection = DebeziumUtils.createMySqlConnection(config); + String writerConnectionString = writerConnection.connectionString(); + + // Both should use the writer hostname when reader is not configured + assertThat(readerConnectionString).contains(WRITER_HOSTNAME); + assertThat(writerConnectionString).contains(WRITER_HOSTNAME); + } + + @Test + void testJdbcConnectionFactoryUsesHostnameOverride() { + MySqlSourceConfig config = createConfig(WRITER_HOSTNAME, null); + + // Factory without hostname override - should use config hostname + JdbcConnectionFactory factoryWithoutOverride = new JdbcConnectionFactory(config); + + // Factory with hostname override - should use override + JdbcConnectionFactory factoryWithOverride = + new JdbcConnectionFactory(config, READER_HOSTNAME); + + // Verify the factories are created with correct configuration + assertThat(factoryWithoutOverride).isNotNull(); + assertThat(factoryWithOverride).isNotNull(); + } Review Comment: `testJdbcConnectionFactoryUsesHostnameOverride()` only asserts that the factory objects are non-null, which doesn't validate the new hostname override behavior. Consider asserting something observable (e.g., that the generated pool name/JDBC URL uses the override hostname, or that the computed `ConnectionPoolId` differs) so this test would fail if override routing regresses. ########## flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/source/config/MySqlSourceConfigFactory.java: ########## @@ -84,6 +85,16 @@ public MySqlSourceConfigFactory hostname(String hostname) { return this; } + /** + * Optional hostname of a MySQL read replica to use for snapshot queries. When set, snapshot + * data will be read from this replica instead of the primary writer instance, reducing the load + * on the writer. The binlog position will still be obtained from the primary writer instance. + */ + public MySqlSourceConfigFactory snapshotHostname(String snapshotHostname) { + this.snapshotHostname = snapshotHostname; Review Comment: `snapshotHostname` is accepted as-is; if a user configures it as an empty/blank string, the connector will attempt to connect to an invalid host instead of cleanly falling back to the primary. Consider normalizing blank values to `null` (or validating and failing fast with a clear message) when setting/creating the config. ```suggestion * * <p>Blank or whitespace-only values are treated as {@code null} so that the connector falls * back to using the primary writer instance for snapshots. */ public MySqlSourceConfigFactory snapshotHostname(String snapshotHostname) { if (snapshotHostname != null && snapshotHostname.trim().isEmpty()) { this.snapshotHostname = null; } else { this.snapshotHostname = snapshotHostname; } ``` ########## flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/pom.xml: ########## @@ -185,6 +185,13 @@ limitations under the License. <!-- test dependencies on TestContainers --> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <version>3.4.6</version> + <scope>test</scope> + </dependency> + <dependency> Review Comment: This module now declares `mockito-core` twice (once pinned to `3.4.6` and again using `${mockito.version}`), which can cause dependency convergence issues and unpredictable Mockito behavior at test runtime. Keep a single `mockito-core` dependency and use the existing `${mockito.version}` property (or remove the property and manage the version in one place). ```suggestion ``` ########## flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/task/MySqlSnapshotSplitReadTaskConnectionTest.java: ########## @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.cdc.connectors.mysql.debezium.task; + +import org.apache.flink.cdc.connectors.mysql.debezium.DebeziumUtils; +import org.apache.flink.cdc.connectors.mysql.debezium.task.context.StatefulTaskContext; +import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig; +import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfigFactory; +import org.apache.flink.cdc.connectors.mysql.table.StartupOptions; + +import com.github.shyiko.mysql.binlog.BinaryLogClient; +import io.debezium.connector.mysql.MySqlConnection; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.time.ZoneId; +import java.util.Properties; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * Tests for {@link MySqlSnapshotSplitReadTask} to verify correct usage of writer and reader + * connections. + * + * <p>The key behavior being tested: + * + * <ul> + * <li>Writer connection is used for binlog position retrieval (SHOW MASTER STATUS) + * <li>Reader connection is used for snapshot data queries (SELECT statements) + * <li>When reader hostname is not configured, reader connection falls back to writer + * </ul> + */ +class MySqlSnapshotSplitReadTaskConnectionTest { + + private static final String PRIMARY_HOSTNAME = "writer-host"; + private static final String SNAPSHOT_HOSTNAME = "reader-host"; + private static final int PORT = 3306; + + @Test + void testStatefulTaskContextWithSeparateConnections() { + MySqlSourceConfig config = createConfig(PRIMARY_HOSTNAME, SNAPSHOT_HOSTNAME); + BinaryLogClient binaryLogClient = mock(BinaryLogClient.class); + + // Create separate writer and reader connections + MySqlConnection writerConnection = DebeziumUtils.createMySqlConnection(config); + MySqlConnection readerConnection = DebeziumUtils.createSnapshotMySqlConnection(config); + + // Create StatefulTaskContext with both connections + StatefulTaskContext context = + new StatefulTaskContext(config, binaryLogClient, writerConnection, readerConnection); + + // Verify the context returns the correct connections + assertThat(context.getConnection()).isSameAs(writerConnection); + assertThat(context.getSnapshotConnection()).isSameAs(readerConnection); + + // Verify writer and reader connections point to different hosts + assertThat(writerConnection.connectionString()).contains(PRIMARY_HOSTNAME); + assertThat(readerConnection.connectionString()).contains(SNAPSHOT_HOSTNAME); + } + + @Test + void testStatefulTaskContextWithSameConnectionWhenNoReaderConfigured() { + MySqlSourceConfig config = createConfig(PRIMARY_HOSTNAME, null); + BinaryLogClient binaryLogClient = mock(BinaryLogClient.class); + + // When no reader hostname is configured, createReaderMySqlConnection falls back to writer + MySqlConnection writerConnection = DebeziumUtils.createMySqlConnection(config); + MySqlConnection readerConnection = DebeziumUtils.createSnapshotMySqlConnection(config); + + // Create StatefulTaskContext + StatefulTaskContext context = + new StatefulTaskContext(config, binaryLogClient, writerConnection, readerConnection); + + // Verify both connections point to the same host (writer) + assertThat(context.getConnection().connectionString()).contains(PRIMARY_HOSTNAME); + assertThat(context.getSnapshotConnection().connectionString()).contains(PRIMARY_HOSTNAME); + } + + @Test + void testStatefulTaskContextLegacyConstructorUsesSameConnection() { + MySqlSourceConfig config = createConfig(PRIMARY_HOSTNAME, SNAPSHOT_HOSTNAME); + BinaryLogClient binaryLogClient = mock(BinaryLogClient.class); + + // Use the legacy constructor that takes only one connection + MySqlConnection connection = DebeziumUtils.createMySqlConnection(config); + StatefulTaskContext context = + new StatefulTaskContext(config, binaryLogClient, connection); + + // Both getConnection() and getReaderConnection() should return the same instance Review Comment: This comment references `getReaderConnection()`, but the API under test is `getSnapshotConnection()`. Update the wording to match the actual method name to avoid confusion when reading/maintaining the test. ```suggestion // Both getConnection() and getSnapshotConnection() should return the same instance ``` ########## flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql/src/main/java/org/apache/flink/cdc/connectors/mysql/source/MySqlDataSourceOptions.java: ########## @@ -93,6 +93,21 @@ public class MySqlDataSourceOptions { .withDescription( "The chunk size (number of rows) of table snapshot, captured tables are split into multiple chunks when read the snapshot of table."); + public static final ConfigOption<String> SCAN_SNAPSHOT_HOSTNAME = + ConfigOptions.key("scan.snapshot.hostname") + .stringType() + .noDefaultValue() + .withDescription( + "Optional IP address or hostname of a MySQL read replica to use for snapshot and metadata" + + "queries (table discovery, chunk splitting). When set, all snapshot data reads and metadata" + + "operations are routed to this instance, reducing load on the primary writer." + + "Binlog position tracking and changelog streaming always use the primary writer instance." + + "WARNING: At-least-once semantics only. When this option is set, exactly-once" + + "guarantees cannot be maintained because the binlog positions recorded during snapshot" + + "scanning originate from the primary writer while the data is read from the replica, and" + + "storage replication lag (even in the millisecond range, as with Aurora/RDS) means the" + + "two positions may not be perfectly consistent."); Review Comment: The `withDescription(...)` string concatenations are missing spaces at the join points (e.g., "metadata" + "queries" and "writer." + "Binlog"), which will render as run-on text in generated docs/help output. Add explicit trailing/leading spaces (or use a single multi-line string) so the description reads correctly. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
