This is an automated email from the ASF dual-hosted git repository.
corgy pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new ecff2e8618 [Feature][Connector-V2] Iceberg add glue catalog support
(#9247)
ecff2e8618 is described below
commit ecff2e861872832b53a83e4a292d3463d0027444
Author: itnccuong <[email protected]>
AuthorDate: Sat May 17 08:42:22 2025 +0700
[Feature][Connector-V2] Iceberg add glue catalog support (#9247)
---
docs/en/connector-v2/sink/Iceberg.md | 79 +++++++++++++++--------
seatunnel-connectors-v2/connector-iceberg/pom.xml | 25 ++++++-
2 files changed, 76 insertions(+), 28 deletions(-)
diff --git a/docs/en/connector-v2/sink/Iceberg.md
b/docs/en/connector-v2/sink/Iceberg.md
index 990b4cc073..9433d12dbf 100644
--- a/docs/en/connector-v2/sink/Iceberg.md
+++ b/docs/en/connector-v2/sink/Iceberg.md
@@ -133,48 +133,75 @@ sink {
```hocon
sink {
Iceberg {
- catalog_name="seatunnel_test"
- iceberg.catalog.config={
+ catalog_name = "seatunnel_test"
+ iceberg.catalog.config = {
type = "hive"
uri = "thrift://localhost:9083"
- warehouse = "hdfs://your_cluster//tmp/seatunnel/iceberg/"
+ warehouse = "hdfs://your_cluster/tmp/seatunnel/iceberg/"
}
- namespace="seatunnel_namespace"
- table="iceberg_sink_table"
- iceberg.table.write-props={
- write.format.default="parquet"
- write.target-file-size-bytes=536870912
+ namespace = "seatunnel_namespace"
+ table = "iceberg_sink_table"
+ iceberg.table.write-props = {
+ write.format.default = "parquet"
+ write.target-file-size-bytes = 536870912
}
- iceberg.table.primary-keys="id"
- iceberg.table.partition-keys="f_datetime"
- iceberg.table.upsert-mode-enabled=true
- iceberg.table.schema-evolution-enabled=true
- case_sensitive=true
+ iceberg.table.primary-keys = "id"
+ iceberg.table.partition-keys = "f_datetime"
+ iceberg.table.upsert-mode-enabled = true
+ iceberg.table.schema-evolution-enabled = true
+ case_sensitive = true
}
}
```
-### Hadoop catalog
+### Hadoop Catalog
```hocon
sink {
Iceberg {
- catalog_name="seatunnel_test"
- iceberg.catalog.config={
+ catalog_name = "seatunnel_test"
+ iceberg.catalog.config = {
type = "hadoop"
warehouse = "hdfs://your_cluster/tmp/seatunnel/iceberg/"
}
- namespace="seatunnel_namespace"
- table="iceberg_sink_table"
- iceberg.table.write-props={
- write.format.default="parquet"
- write.target-file-size-bytes=536870912
+ namespace = "seatunnel_namespace"
+ table = "iceberg_sink_table"
+ iceberg.table.write-props = {
+ write.format.default = "parquet"
+ write.target-file-size-bytes = 536870912
}
- iceberg.table.primary-keys="id"
- iceberg.table.partition-keys="f_datetime"
- iceberg.table.upsert-mode-enabled=true
- iceberg.table.schema-evolution-enabled=true
- case_sensitive=true
+ iceberg.table.primary-keys = "id"
+ iceberg.table.partition-keys = "f_datetime"
+ iceberg.table.upsert-mode-enabled = true
+ iceberg.table.schema-evolution-enabled = true
+ case_sensitive = true
+ }
+}
+```
+
+### Glue Catalog
+
+```hocon
+sink {
+ Iceberg {
+ catalog_name = "seatunnel_test"
+ iceberg.catalog.config = {
+ warehouse = "s3://your-bucket/warehouse/"
+ catalog-impl = "org.apache.iceberg.aws.glue.GlueCatalog"
+ io-impl = "org.apache.iceberg.aws.s3.S3FileIO"
+ client.region = "your-region"
+ }
+ namespace = "seatunnel_namespace"
+ table = "iceberg_sink_table"
+ iceberg.table.write-props = {
+ write.format.default = "parquet"
+ write.target-file-size-bytes = 536870912
+ }
+ iceberg.table.primary-keys = "id"
+ iceberg.table.partition-keys = "f_datetime"
+ iceberg.table.upsert-mode-enabled = true
+ iceberg.table.schema-evolution-enabled = true
+ case_sensitive = true
}
}
diff --git a/seatunnel-connectors-v2/connector-iceberg/pom.xml
b/seatunnel-connectors-v2/connector-iceberg/pom.xml
index fd8efba245..88632ec824 100644
--- a/seatunnel-connectors-v2/connector-iceberg/pom.xml
+++ b/seatunnel-connectors-v2/connector-iceberg/pom.xml
@@ -35,6 +35,7 @@
<avro.version>1.11.3</avro.version>
<hive.version>2.3.9</hive.version>
<connector.name>connector-iceberg</connector.name>
+
<software.amazon.awssdk.version>2.31.30</software.amazon.awssdk.version>
</properties>
<dependencyManagement>
@@ -58,19 +59,38 @@
<artifactId>jsqlparser</artifactId>
<version>${jsqlparser.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.iceberg</groupId>
+ <artifactId>iceberg-aws</artifactId>
+ <version>${iceberg.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>glue</artifactId>
+ <version>${software.amazon.awssdk.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ <version>${software.amazon.awssdk.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>sts</artifactId>
+ <version>${software.amazon.awssdk.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-core</artifactId>
<version>${iceberg.version}</version>
</dependency>
-
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-common</artifactId>
<version>${iceberg.version}</version>
</dependency>
-
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-api</artifactId>
@@ -229,6 +249,7 @@
</goals>
<phase>package</phase>
<configuration>
+ <minimizeJar>false</minimizeJar>
<relocations>
<relocation>
<pattern>org.apache.avro</pattern>
