This is an automated email from the ASF dual-hosted git repository.
zirui pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
from 9bcbb3916 [INLONG-6787][Doc] Update the readme for Apache Hudi (#6812)
add 4206c99c9 [INLONG-6735][Sort][Manager] PostgreSQL connector supports
transferring all tables for all schemas in one database (#6736)
No new revisions were added by this update.
Summary of changes:
.../inlong/manager/common/enums/FieldType.java | 2 +-
.../manager/pojo/sort/util/ExtractNodeUtils.java | 22 +-
.../manager/pojo/sort/util/FieldInfoUtils.java | 1 +
.../pojo/source/postgresql/PostgreSQLSource.java | 6 +
.../source/postgresql/PostgreSQLSourceDTO.java | 6 +
.../source/postgresql/PostgreSQLSourceRequest.java | 6 +
.../sort/protocol/constant/PostgresConstant.java | 4 +
.../protocol/node/extract/PostgresExtractNode.java | 79 +++-
.../node/extract/PostgresExtractNodeTest.java | 22 +-
inlong-sort/sort-connectors/base/pom.xml | 1 +
.../sort/base/format/JsonDynamicSchemaFormat.java | 10 +
.../sort-connectors/{base => cdc-base}/pom.xml | 15 +-
.../debezium/DebeziumDeserializationSchema.java | 5 +-
.../history/FlinkJsonTableChangeSerializer.java | 3 +-
.../base}/debezium/internal/DebeziumOffset.java | 5 +-
.../internal/DebeziumOffsetSerializer.java | 5 +-
.../debezium/internal/FlinkDatabaseHistory.java | 14 +-
.../internal/FlinkDatabaseSchemaHistory.java | 21 +-
.../debezium/internal/FlinkOffsetBackingStore.java | 24 +-
.../sort/cdc/base}/debezium/internal/Handover.java | 17 +-
.../cdc/base}/debezium/internal/SchemaRecord.java | 5 +-
.../debezium/table/AppendMetadataCollector.java | 5 +-
.../cdc/base}/debezium/table/DebeziumOptions.java | 2 +-
.../table/DeserializationRuntimeConverter.java | 7 +-
.../DeserializationRuntimeConverterFactory.java | 5 +-
.../base}/debezium/table/MetadataConverter.java | 7 +-
.../table/RowDataDebeziumDeserializeSchema.java | 47 +-
.../sort/cdc/base/util}/DatabaseHistoryUtil.java | 5 +-
.../inlong/sort/cdc/base/util}/RecordUtils.java | 2 +-
.../sort/cdc/base/util}/TemporalConversions.java | 2 +-
inlong-sort/sort-connectors/mysql-cdc/pom.xml | 5 +
.../sort/cdc/debezium/DebeziumSourceFunction.java | 26 +-
.../JsonDebeziumDeserializationSchema.java | 1 +
.../StringDebeziumDeserializationSchema.java | 1 +
.../debezium/internal/DebeziumChangeConsumer.java | 2 +
.../debezium/internal/DebeziumChangeFetcher.java | 9 +-
.../apache/inlong/sort/cdc/mysql/MySqlSource.java | 4 +-
.../cdc/mysql/SeekBinlogToTimestampFilter.java | 2 +-
.../inlong/sort/cdc/mysql/source/MySqlSource.java | 2 +-
.../sort/cdc/mysql/source/MySqlSourceBuilder.java | 2 +-
.../mysql/source/reader/MySqlRecordEmitter.java | 4 +-
.../mysql/source/split/MySqlSplitSerializer.java | 2 +-
.../MySqlDeserializationConverterFactory.java | 4 +-
.../cdc/mysql/table/MySqlReadableMetadata.java | 2 +-
.../mysql/table/MySqlTableInlongSourceFactory.java | 4 +-
.../sort/cdc/mysql/table/MySqlTableSource.java | 6 +-
.../cdc/mysql/table/OldFieldMetadataConverter.java | 4 +-
inlong-sort/sort-connectors/oracle-cdc/pom.xml | 5 +
.../inlong/sort/cdc/oracle/OracleSource.java | 4 +-
.../debezium/DebeziumDeserializationSchema.java | 52 ---
.../oracle/debezium/DebeziumSourceFunction.java | 17 +-
.../history/FlinkJsonTableChangeSerializer.java | 207 --------
.../debezium/internal/DebeziumChangeConsumer.java | 2 +
.../debezium/internal/DebeziumChangeFetcher.java | 10 +-
.../oracle/debezium/internal/DebeziumOffset.java | 65 ---
.../internal/DebeziumOffsetSerializer.java | 42 --
.../debezium/internal/FlinkDatabaseHistory.java | 118 -----
.../internal/FlinkDatabaseSchemaHistory.java | 8 +-
.../debezium/internal/FlinkOffsetBackingStore.java | 200 --------
.../cdc/oracle/debezium/internal/Handover.java | 194 --------
.../cdc/oracle/debezium/internal/SchemaRecord.java | 97 ----
.../debezium/table/AppendMetadataCollector.java | 73 ---
.../cdc/oracle/debezium/table/DebeziumOptions.java | 54 ---
.../DeserializationRuntimeConverterFactory.java | 47 --
.../oracle/debezium/table/MetadataConverter.java | 44 --
.../table/RowDataDebeziumDeserializeSchema.java | 10 +-
.../oracle/debezium/utils/DatabaseHistoryUtil.java | 87 ----
.../oracle/debezium/utils/TemporalConversions.java | 231 ---------
.../OracleDeserializationConverterFactory.java | 4 +-
.../cdc/oracle/table/OracleReadableMetaData.java | 2 +-
.../sort/cdc/oracle/table/OracleTableSource.java | 4 +-
.../cdc/oracle/table/OracleTableSourceFactory.java | 2 +-
inlong-sort/sort-connectors/pom.xml | 1 +
inlong-sort/sort-connectors/postgres-cdc/pom.xml | 40 +-
.../DebeziumSourceFunction.java | 90 ++--
.../PostgreSQLSource.java | 2 +-
.../PostgreSQLJdbcConnectionIProvider.java} | 21 +-
.../PostgreSQLJdbcConnectionOptions.java | 64 +++
.../PostgreSQLJdbcConnectionProvider.java | 83 ++++
.../debezium/internal/ColumnImpl.java | 236 ++++++++++
.../debezium/internal/DebeziumChangeFetcher.java | 207 ++++++--
.../debezium/internal/TableEditorImpl.java | 278 +++++++++++
.../debezium/internal/TableImpl.java | 143 ++++++
.../manager/PostgreSQLQueryVisitor.java | 195 ++++++++
.../cdc/postgres/table/PostgreSQLDataType.java | 73 +++
...PostgreSQLDeserializationConverterFactory.java} | 50 +-
.../postgres/table/PostgreSQLReadableMetaData.java | 520 +++++++++++++++++++++
.../cdc/postgres/table/PostgreSQLTableFactory.java | 47 ++
.../cdc/postgres/table/PostgreSQLTableSource.java | 82 ++--
.../sort/parser/AllMigratePostgreSQLTest.java | 125 +++++
.../parser/PostgresExtractFlinkSqlParseTest.java | 2 +-
licenses/inlong-sort-connectors/LICENSE | 65 ++-
92 files changed, 2452 insertions(+), 1886 deletions(-)
copy inlong-sort/sort-connectors/{base => cdc-base}/pom.xml (78%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/DebeziumDeserializationSchema.java
(97%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/history/FlinkJsonTableChangeSerializer.java
(99%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/internal/DebeziumOffset.java
(97%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/internal/DebeziumOffsetSerializer.java
(96%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/internal/FlinkDatabaseHistory.java
(88%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/internal/FlinkDatabaseSchemaHistory.java
(91%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/internal/FlinkOffsetBackingStore.java
(98%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/internal/Handover.java
(99%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/internal/SchemaRecord.java
(98%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/table/AppendMetadataCollector.java
(97%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/table/DebeziumOptions.java
(97%)
rename
inlong-sort/sort-connectors/{oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/table/DeserializationRuntimeConverter.java
(85%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/table/DeserializationRuntimeConverterFactory.java
(97%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/table/MetadataConverter.java
(96%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base}/debezium/table/RowDataDebeziumDeserializeSchema.java
(97%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/debezium/utils
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base/util}/DatabaseHistoryUtil.java
(96%)
rename
inlong-sort/sort-connectors/{oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/utils
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base/util}/RecordUtils.java
(98%)
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/debezium/utils
=>
cdc-base/src/main/java/org/apache/inlong/sort/cdc/base/util}/TemporalConversions.java
(99%)
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/DebeziumDeserializationSchema.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/history/FlinkJsonTableChangeSerializer.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/internal/DebeziumOffset.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/internal/DebeziumOffsetSerializer.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/internal/FlinkDatabaseHistory.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/internal/FlinkOffsetBackingStore.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/internal/Handover.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/internal/SchemaRecord.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/table/AppendMetadataCollector.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/table/DebeziumOptions.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/table/DeserializationRuntimeConverterFactory.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/table/MetadataConverter.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/utils/DatabaseHistoryUtil.java
delete mode 100644
inlong-sort/sort-connectors/oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/debezium/utils/TemporalConversions.java
rename
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/debezium/table/DeserializationRuntimeConverter.java
=>
postgres-cdc/src/main/java/org.apache.inlong.sort.cdc.postgres/connection/PostgreSQLJdbcConnectionIProvider.java}
(67%)
create mode 100644
inlong-sort/sort-connectors/postgres-cdc/src/main/java/org.apache.inlong.sort.cdc.postgres/connection/PostgreSQLJdbcConnectionOptions.java
create mode 100644
inlong-sort/sort-connectors/postgres-cdc/src/main/java/org.apache.inlong.sort.cdc.postgres/connection/PostgreSQLJdbcConnectionProvider.java
create mode 100644
inlong-sort/sort-connectors/postgres-cdc/src/main/java/org.apache.inlong.sort.cdc.postgres/debezium/internal/ColumnImpl.java
copy
inlong-sort/sort-connectors/{mysql-cdc/src/main/java/org/apache/inlong/sort/cdc
=>
postgres-cdc/src/main/java/org.apache.inlong.sort.cdc.postgres}/debezium/internal/DebeziumChangeFetcher.java
(56%)
create mode 100644
inlong-sort/sort-connectors/postgres-cdc/src/main/java/org.apache.inlong.sort.cdc.postgres/debezium/internal/TableEditorImpl.java
create mode 100644
inlong-sort/sort-connectors/postgres-cdc/src/main/java/org.apache.inlong.sort.cdc.postgres/debezium/internal/TableImpl.java
create mode 100644
inlong-sort/sort-connectors/postgres-cdc/src/main/java/org.apache.inlong.sort.cdc.postgres/manager/PostgreSQLQueryVisitor.java
create mode 100644
inlong-sort/sort-connectors/postgres-cdc/src/main/java/org/apache/inlong/sort/cdc/postgres/table/PostgreSQLDataType.java
copy
inlong-sort/sort-connectors/{oracle-cdc/src/main/java/org/apache/inlong/sort/cdc/oracle/table/OracleDeserializationConverterFactory.java
=>
postgres-cdc/src/main/java/org/apache/inlong/sort/cdc/postgres/table/PostgreSQLDeserializationConverterFactory.java}
(81%)
create mode 100644
inlong-sort/sort-connectors/postgres-cdc/src/main/java/org/apache/inlong/sort/cdc/postgres/table/PostgreSQLReadableMetaData.java
create mode 100644
inlong-sort/sort-core/src/test/java/org/apache/inlong/sort/parser/AllMigratePostgreSQLTest.java
