This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
from c946eb797 [Feat][UI] Update user service in the project. (#2756)
add ffc6088a7 [Feature][Connector-V2] Add iceberg source connector (#2615)
No new revisions were added by this update.
Summary of changes:
LICENSE | 6 +-
NOTICE | 19 +-
docs/en/connector-v2/source/Iceberg.md | 157 ++++++++++++++++
plugin-mapping.properties | 2 +-
.../seatunnel/console/sink/ConsoleSinkWriter.java | 3 +
seatunnel-connectors-v2/connector-iceberg/pom.xml | 193 +++++++++++++++++++
.../seatunnel/iceberg/IcebergCatalogFactory.java | 82 ++++++++
.../seatunnel/iceberg/IcebergTableLoader.java | 77 ++++++++
.../seatunnel/iceberg/config/CommonConfig.java | 79 ++++++++
.../seatunnel/iceberg/config/SourceConfig.java | 81 ++++++++
.../iceberg/data/DefaultDeserializer.java | 139 ++++++++++++++
.../seatunnel/iceberg/data/Deserializer.java} | 8 +-
.../iceberg/data/IcebergRecordProjection.java | 97 ++++++++++
.../seatunnel/iceberg/data/IcebergTypeMapper.java | 111 +++++++++++
.../seatunnel/iceberg/source/IcebergSource.java | 178 ++++++++++++++++++
.../source/enumerator/AbstractSplitEnumerator.java | 130 +++++++++++++
.../enumerator/IcebergBatchSplitEnumerator.java | 71 +++++++
.../enumerator/IcebergEnumerationResult.java} | 19 +-
.../enumerator/IcebergEnumeratorPosition.java} | 26 ++-
.../enumerator/IcebergSplitEnumeratorState.java} | 21 ++-
.../enumerator/IcebergStreamSplitEnumerator.java | 85 +++++++++
.../source/enumerator/scan/IcebergScanContext.java | 85 +++++++++
.../enumerator/scan/IcebergScanSplitPlanner.java | 207 +++++++++++++++++++++
.../scan/IcebergStreamScanStrategy.java} | 31 +--
.../source/reader/IcebergFileScanTaskReader.java | 143 ++++++++++++++
.../reader/IcebergFileScanTaskSplitReader.java | 86 +++++++++
.../iceberg/source/reader/IcebergSourceReader.java | 145 +++++++++++++++
.../source/split/IcebergFileScanTaskSplit.java | 68 +++++++
.../seatunnel/iceberg/TestIcebergMetastore.java | 63 +++++++
seatunnel-connectors-v2/pom.xml | 1 +
seatunnel-dist/pom.xml | 6 +
.../pom.xml | 13 +-
.../e2e/flink/v2/iceberg/IcebergSourceIT.java | 165 ++++++++++++++++
.../src/test/resources/iceberg/iceberg_source.conf | 82 ++++++++
.../src/test/resources/log4j.properties | 0
.../seatunnel-flink-connector-v2-e2e/pom.xml | 1 +
.../pom.xml | 12 +-
.../e2e/flink/v2/iceberg/IcebergSourceIT.java | 164 ++++++++++++++++
.../src/test/resources/iceberg/iceberg_source.conf | 85 +++++++++
.../src/test/resources/log4j.properties | 0
40 files changed, 2868 insertions(+), 73 deletions(-)
create mode 100644 docs/en/connector-v2/source/Iceberg.md
create mode 100644 seatunnel-connectors-v2/connector-iceberg/pom.xml
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/IcebergCatalogFactory.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/IcebergTableLoader.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/config/CommonConfig.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/config/SourceConfig.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/DefaultDeserializer.java
copy
seatunnel-connectors-v2/{connector-elasticsearch/src/java/org/apache/seatunnel/connectors/seatunnel/elasticsearch/serialize/SeaTunnelRowSerializer.java
=>
connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/Deserializer.java}
(82%)
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/IcebergRecordProjection.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/data/IcebergTypeMapper.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/IcebergSource.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/enumerator/AbstractSplitEnumerator.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/enumerator/IcebergBatchSplitEnumerator.java
copy
seatunnel-connectors-v2/{connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/serialize/IoTDBRecord.java
=>
connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/enumerator/IcebergEnumerationResult.java}
(68%)
copy
seatunnel-connectors-v2/{connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/FileCommitInfo.java
=>
connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/enumerator/IcebergEnumeratorPosition.java}
(66%)
copy
seatunnel-connectors-v2/{connector-iotdb/src/main/java/org/apache/seatunnel/connectors/seatunnel/iotdb/config/CommonConfig.java
=>
connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/enumerator/IcebergSplitEnumeratorState.java}
(63%)
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/enumerator/IcebergStreamSplitEnumerator.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/enumerator/scan/IcebergScanContext.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/enumerator/scan/IcebergScanSplitPlanner.java
copy
seatunnel-connectors-v2/{connector-kafka/src/main/java/org/apache/seatunnel/connectors/seatunnel/kafka/config/KafkaSemantics.java
=>
connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/enumerator/scan/IcebergStreamScanStrategy.java}
(56%)
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/reader/IcebergFileScanTaskReader.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/reader/IcebergFileScanTaskSplitReader.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/reader/IcebergSourceReader.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/main/java/org/apache/seatunnel/connectors/seatunnel/iceberg/source/split/IcebergFileScanTaskSplit.java
create mode 100644
seatunnel-connectors-v2/connector-iceberg/src/test/java/org/apache/seatunnel/connectors/seatunnel/iceberg/TestIcebergMetastore.java
copy seatunnel-e2e/seatunnel-flink-connector-v2-e2e/{connector-iotdb-flink-e2e
=> connector-iceberg-flink-e2e}/pom.xml (82%)
create mode 100644
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-iceberg-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/iceberg/IcebergSourceIT.java
create mode 100644
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-iceberg-flink-e2e/src/test/resources/iceberg/iceberg_source.conf
copy
seatunnel-e2e/seatunnel-flink-connector-v2-e2e/{connector-assert-flink-e2e =>
connector-iceberg-flink-e2e}/src/test/resources/log4j.properties (100%)
copy seatunnel-e2e/seatunnel-spark-connector-v2-e2e/{connector-iotdb-spark-e2e
=> connector-iceberg-spark-e2e}/pom.xml (82%)
create mode 100644
seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-iceberg-spark-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/iceberg/IcebergSourceIT.java
create mode 100644
seatunnel-e2e/seatunnel-spark-connector-v2-e2e/connector-iceberg-spark-e2e/src/test/resources/iceberg/iceberg_source.conf
copy
seatunnel-e2e/{seatunnel-flink-connector-v2-e2e/connector-assert-flink-e2e =>
seatunnel-spark-connector-v2-e2e/connector-iceberg-spark-e2e}/src/test/resources/log4j.properties
(100%)
