This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
from f99f280286 [Improve] Remove checkpoint timeout check if checkpoint
disabled (#6231)
add 97438b9402 [Feature][Connector-V2][Postgres-cdc]Support for Postgres
cdc (#5986)
No new revisions were added by this update.
Summary of changes:
docs/en/connector-v2/source/Postgre-CDC.md | 188 +++
plugin-mapping.properties | 1 +
.../cdc/base/dialect/DataSourceDialect.java | 13 +-
.../cdc/base/source/IncrementalSource.java | 1 +
.../source/reader/IncrementalSourceReader.java | 9 +-
.../external/JdbcSourceFetchTaskContext.java | 4 +-
.../pom.xml | 38 +-
.../postgresql/PostgresConnectorConfig.java | 1277 ++++++++++++++++++++
.../postgresql/PostgresEventMetadataProvider.java | 85 ++
.../postgresql/PostgresOffsetContext.java | 382 ++++++
.../connector/postgresql/PostgresSchema.java | 310 +++++
.../connector/postgresql/PostgresTaskContext.java | 141 +++
.../connector/postgresql/TypeRegistry.java | 487 ++++++++
.../postgresql/connection/PostgresConnection.java | 769 ++++++++++++
.../cdc/postgres/config/PostgresSourceConfig.java} | 17 +-
.../config/PostgresSourceConfigFactory.java} | 82 +-
.../cdc/postgres/option/PostgresOptions.java | 50 +
.../cdc/postgres/source/PostgresDialect.java | 176 +++
.../source/PostgresIncrementalSource.java} | 47 +-
.../source/PostgresIncrementalSourceFactory.java} | 37 +-
.../source/PostgresPooledDataSourceFactory.java} | 8 +-
.../postgres/source/PostgresSourceOptions.java} | 9 +-
.../source/enumerator/PostgresChunkSplitter.java} | 24 +-
.../cdc/postgres/source/offset/LsnOffset.java | 124 ++
.../postgres}/source/offset/LsnOffsetFactory.java | 25 +-
.../reader/PostgresSourceFetchTaskContext.java | 403 ++++++
.../snapshot/PostgresSnapshotFetchTask.java} | 96 +-
.../snapshot/PostgresSnapshotSplitReadTask.java} | 47 +-
.../SnapshotSplitChangeEventSourceContext.java | 4 +-
.../source/reader/wal/PostgresWalFetchTask.java | 117 ++
.../postgres/utils/PostgresConnectionUtils.java | 44 +
.../cdc/postgres/utils/PostgresSchema.java} | 39 +-
.../cdc/postgres/utils/PostgresTypeUtils.java} | 47 +-
.../cdc/postgres/utils/PostgresUtils.java} | 69 +-
.../cdc/postgres}/utils/TableDiscoveryUtils.java | 11 +-
seatunnel-connectors-v2/connector-cdc/pom.xml | 1 +
.../internal/dialect/psql/PostgresDialect.java | 1 +
seatunnel-dist/pom.xml | 6 +
.../pom.xml | 17 +-
.../seatunnel/cdc/postgres/PostgresCDCIT.java | 585 +++++++++
.../src/test/resources/ddl/inventory.sql | 176 +++
...cdc_to_pg_with_multi_table_mode_one_table.conf} | 36 +-
...cdc_to_pg_with_multi_table_mode_two_table.conf} | 36 +-
.../test/resources/postgrescdc_to_postgres.conf} | 35 +-
...escdc_to_postgres_with_custom_primary_key.conf} | 40 +-
...stgrescdc_to_postgres_with_no_primary_key.conf} | 34 +-
seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml | 1 +
47 files changed, 5716 insertions(+), 433 deletions(-)
create mode 100644 docs/en/connector-v2/source/Postgre-CDC.md
copy seatunnel-connectors-v2/connector-cdc/{connector-cdc-mysql =>
connector-cdc-postgres}/pom.xml (75%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/io/debezium/connector/postgresql/PostgresConnectorConfig.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/io/debezium/connector/postgresql/PostgresEventMetadataProvider.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/io/debezium/connector/postgresql/PostgresOffsetContext.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/io/debezium/connector/postgresql/PostgresSchema.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/io/debezium/connector/postgresql/PostgresTaskContext.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/io/debezium/connector/postgresql/TypeRegistry.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/io/debezium/connector/postgresql/connection/PostgresConnection.java
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/config/MySqlSourceConfig.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/config/PostgresSourceConfig.java}
(84%)
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/config/SqlServerSourceConfigFactory.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/config/PostgresSourceConfigFactory.java}
(50%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/option/PostgresOptions.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/PostgresDialect.java
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/source/OracleIncrementalSource.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/PostgresIncrementalSource.java}
(76%)
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactory.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/PostgresIncrementalSourceFactory.java}
(74%)
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/source/SqlServerPooledDataSourceFactory.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/PostgresPooledDataSourceFactory.java}
(79%)
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/source/SqlServerSourceOptions.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/PostgresSourceOptions.java}
(88%)
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/source/eumerator/SqlServerChunkSplitter.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/enumerator/PostgresChunkSplitter.java}
(74%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/offset/LsnOffset.java
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres}/source/offset/LsnOffsetFactory.java
(67%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/reader/PostgresSourceFetchTaskContext.java
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/reader/fetch/scan/MySqlSnapshotFetchTask.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/reader/snapshot/PostgresSnapshotFetchTask.java}
(52%)
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/source/reader/fetch/scan/SqlServerSnapshotSplitReadTask.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/reader/snapshot/PostgresSnapshotSplitReadTask.java}
(87%)
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/source/reader/fetch/scan
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/reader/snapshot}/SnapshotSplitChangeEventSourceContext.java
(90%)
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/source/reader/wal/PostgresWalFetchTask.java
create mode 100644
seatunnel-connectors-v2/connector-cdc/connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/utils/PostgresConnectionUtils.java
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/utils/SqlServerSchema.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/utils/PostgresSchema.java}
(65%)
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-oracle/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/oracle/utils/OracleTypeUtils.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/utils/PostgresTypeUtils.java}
(70%)
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source/utils/SqlServerUtils.java
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/utils/PostgresUtils.java}
(86%)
copy
seatunnel-connectors-v2/connector-cdc/{connector-cdc-sqlserver/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/sqlserver/source
=>
connector-cdc-postgres/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres}/utils/TableDiscoveryUtils.java
(89%)
copy seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e =>
connector-cdc-postgres-e2e}/pom.xml (84%)
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/postgres/PostgresCDCIT.java
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-postgres-e2e/src/test/resources/ddl/inventory.sql
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_disable_exactly_once.conf
=>
connector-cdc-postgres-e2e/src/test/resources/pgcdc_to_pg_with_multi_table_mode_one_table.conf}
(69%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_disable_exactly_once.conf
=>
connector-cdc-postgres-e2e/src/test/resources/pgcdc_to_pg_with_multi_table_mode_two_table.conf}
(67%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_disable_exactly_once.conf
=> connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres.conf}
(69%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_disable_exactly_once.conf
=>
connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_custom_primary_key.conf}
(64%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/{connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql_with_disable_exactly_once.conf
=>
connector-cdc-postgres-e2e/src/test/resources/postgrescdc_to_postgres_with_no_primary_key.conf}
(69%)
