This is an automated email from the ASF dual-hosted git repository.
liugddx pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
from 0c0eb7e41b [Improve][Jdbc] Jdbc truncate table should check table not
database (#7654)
add 0d12520f91 [Feature][Connector-V2] Optimize hudi sink (#7662)
No new revisions were added by this update.
Summary of changes:
docs/en/connector-v2/sink/Hudi.md | 113 ++++++-
docs/zh/connector-v2/sink/Hudi.md | 156 ++++++++--
seatunnel-connectors-v2/connector-hudi/pom.xml | 17 +-
.../seatunnel/hudi/catalog/HudiCatalog.java | 326 ++++++++++++++++++++
.../hudi/catalog/HudiCatalogFactory.java} | 23 +-
.../seatunnel/hudi/config/HudiOptions.java | 91 ++----
.../seatunnel/hudi/config/HudiSinkConfig.java | 58 ++--
.../seatunnel/hudi/config/HudiTableConfig.java | 209 +++++++++++++
.../{HudiOptions.java => HudiTableOptions.java} | 61 ++--
.../seatunnel/hudi/exception/HudiErrorCode.java | 9 +-
.../seatunnel/hudi/sink/HudiClientManager.java | 65 ++++
.../hudi/sink/HudiMultiTableResourceManager.java | 23 +-
.../connectors/seatunnel/hudi/sink/HudiSink.java | 83 ++++-
.../seatunnel/hudi/sink/HudiSinkFactory.java | 127 ++++++--
.../hudi/sink/client/HudiWriteClientProvider.java | 73 +++++
.../sink/client/HudiWriteClientProviderProxy.java | 58 ++++
.../hudi/sink/client/WriteClientProvider.java} | 13 +-
.../sink/commiter/HudiSinkAggregatedCommitter.java | 102 ++++++
.../committer/HudiSinkAggregatedCommitter.java | 132 --------
.../{writer => convert}/AvroSchemaConverter.java | 5 +-
.../hudi/sink/convert/HudiRecordConverter.java | 163 ++++++++++
.../RowDataToAvroConverters.java | 6 +-
.../{ => sink}/state/HudiAggregatedCommitInfo.java | 2 +-
.../hudi/{ => sink}/state/HudiCommitInfo.java | 8 +-
.../hudi/{ => sink}/state/HudiSinkState.java | 2 +-
.../hudi/sink/writer/HudiRecordWriter.java | 271 ++++++++++++++++
.../seatunnel/hudi/sink/writer/HudiSinkWriter.java | 343 +++++----------------
.../seatunnel/hudi/util/HudiCatalogUtil.java} | 24 +-
.../connectors/seatunnel/hudi/util/HudiUtil.java | 93 +++++-
.../connectors/seatunnel/hudi/util/SchemaUtil.java | 147 +++++++++
.../connectors/seatunnel/hudi/HudiTest.java | 4 +-
.../seatunnel/hudi/catalog/HudiCatalogTest.java} | 95 +++---
.../flink/execution/SinkExecuteProcessor.java | 35 ++-
.../flink/execution/SinkExecuteProcessor.java | 35 ++-
.../spark/execution/SinkExecuteProcessor.java | 35 ++-
.../spark/execution/SinkExecuteProcessor.java | 35 ++-
.../connector-hudi-e2e/pom.xml | 28 ++
.../seatunnel/e2e/connector/hudi/HudiIT.java | 58 +++-
.../hudi/{HudiIT.java => HudiMultiTableIT.java} | 94 ++++--
.../hudi/HudiSeatunnelS3MultiTableIT.java | 221 +++++++++++++
.../connector/hudi/HudiSparkS3MultiTableIT.java | 202 ++++++++++++
.../src/test/resources}/core-site.xml | 40 ++-
.../src/test/resources/fake_to_hudi.conf | 4 +
...onf => fake_to_hudi_with_omit_config_item.conf} | 0
.../src/test/resources/multi_fake_to_hudi.conf | 93 ++++++
.../src/test/resources/s3_fake_to_hudi.conf | 94 ++++++
.../container/seatunnel/SeaTunnelContainer.java | 13 +-
.../seatunnel-hadoop3-3.1.4-uber/pom.xml | 1 -
48 files changed, 3089 insertions(+), 801 deletions(-)
create mode 100644
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/catalog/HudiCatalog.java
copy
seatunnel-connectors-v2/{connector-hbase/src/main/java/org/apache/seatunnel/connectors/seatunnel/hbase/catalog/HbaseCatalogFactory.java
=>
connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/catalog/HudiCatalogFactory.java}
(63%)
create mode 100644
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/config/HudiTableConfig.java
copy
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/config/{HudiOptions.java
=> HudiTableOptions.java} (67%)
create mode 100644
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/HudiClientManager.java
copy
seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/sink/inmemory/InMemoryMultiTableResourceManager.java
=>
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/HudiMultiTableResourceManager.java
(64%)
create mode 100644
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/client/HudiWriteClientProvider.java
create mode 100644
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/client/HudiWriteClientProviderProxy.java
copy
seatunnel-connectors-v2/{connector-milvus/src/main/java/org/apache/seatunnel/connectors/seatunnel/milvus/sink/batch/MilvusBatchWriter.java
=>
connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/client/WriteClientProvider.java}
(74%)
create mode 100644
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/commiter/HudiSinkAggregatedCommitter.java
delete mode 100644
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/committer/HudiSinkAggregatedCommitter.java
rename
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/{writer
=> convert}/AvroSchemaConverter.java (97%)
create mode 100644
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/convert/HudiRecordConverter.java
rename
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/{writer
=> convert}/RowDataToAvroConverters.java (98%)
copy
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/{
=> sink}/state/HudiAggregatedCommitInfo.java (94%)
copy
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/{
=> sink}/state/HudiCommitInfo.java (85%)
copy
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/{
=> sink}/state/HudiSinkState.java (93%)
create mode 100644
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/sink/writer/HudiRecordWriter.java
copy
seatunnel-connectors-v2/{connector-milvus/src/main/java/org/apache/seatunnel/connectors/seatunnel/milvus/source/MilvusSourceSplit.java
=>
connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/util/HudiCatalogUtil.java}
(57%)
create mode 100644
seatunnel-connectors-v2/connector-hudi/src/main/java/org/apache/seatunnel/connectors/seatunnel/hudi/util/SchemaUtil.java
copy
seatunnel-connectors-v2/{connector-iceberg/src/test/java/org/apache/seatunnel/connectors/seatunnel/iceberg/catalog/IcebergCatalogTest.java
=>
connector-hudi/src/test/java/org/apache/seatunnel/connectors/seatunnel/hudi/catalog/HudiCatalogTest.java}
(56%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/{HudiIT.java
=> HudiMultiTableIT.java} (52%)
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSeatunnelS3MultiTableIT.java
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSparkS3MultiTableIT.java
copy {seatunnel-connectors-v2/connector-hive/src/test/resources/cos =>
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources}/core-site.xml
(55%)
copy
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/{fake_to_hudi.conf
=> fake_to_hudi_with_omit_config_item.conf} (100%)
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/multi_fake_to_hudi.conf
create mode 100644
seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/s3_fake_to_hudi.conf
