This is an automated email from the ASF dual-hosted git repository. jinsongzhou pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push: new 957e52124 [AMORO-3084] Upgrade to iceberg 1.6.1 (#3071) 957e52124 is described below commit 957e52124af7c7791af4e7196f421311fc55ceeb Author: ConradJam <jam.gz...@gmail.com> AuthorDate: Thu Jan 16 17:00:19 2025 +0800 [AMORO-3084] Upgrade to iceberg 1.6.1 (#3071) [Improvement]: Upgrade to iceberg 1.6.1 (#3084) --- .../amoro/optimizing/IcebergRewriteExecutor.java | 13 ---------- .../org/apache/amoro/table/BasicUnkeyedTable.java | 6 +++++ .../org/apache/iceberg/TestIcebergFindFiles.java | 28 ++++++++++++++++++---- .../amoro-mixed-flink-common/pom.xml | 3 ++- .../v1.15/amoro-mixed-flink-1.15/pom.xml | 1 + .../v1.16/amoro-mixed-flink-1.16/pom.xml | 3 ++- .../v1.17/amoro-mixed-flink-1.17/pom.xml | 3 ++- .../amoro-mixed-spark-3-common/pom.xml | 3 ++- .../v3.2/amoro-mixed-spark-3.2/pom.xml | 1 + .../v3.3/amoro-mixed-spark-3.3/pom.xml | 1 + pom.xml | 6 ++--- 11 files changed, 44 insertions(+), 24 deletions(-) diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java index fa355a8bf..d17222172 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java @@ -24,7 +24,6 @@ import org.apache.amoro.io.writer.IcebergFanoutPosDeleteWriter; import org.apache.amoro.table.MixedTable; import org.apache.amoro.utils.map.StructLikeCollections; import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.StructLike; import org.apache.iceberg.TableProperties; import org.apache.iceberg.data.GenericAppenderFactory; import org.apache.iceberg.data.IdentityPartitionConverters; @@ -46,18 +45,6 @@ public class IcebergRewriteExecutor extends AbstractRewriteFilesExecutor { super(input, table, structLikeCollections); } - // TODO We can remove this override method after upgrading Iceberg version to 1.5+. - @Override - protected StructLike partition() { - StructLike partitionData = super.partition(); - if (partitionData != null && partitionData.size() == 0) { - // Cast empty partition data to NULL to avoid creating empty partition directory. - return null; - } else { - return partitionData; - } - } - @Override protected OptimizingDataReader dataReader() { return new GenericCombinedIcebergDataReader( diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java index d5f6f878d..6c5cceee6 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java @@ -65,6 +65,7 @@ import org.apache.iceberg.util.StructLikeMap; import java.util.List; import java.util.Map; +import java.util.UUID; /** Basic implementation of {@link UnkeyedTable}, wrapping a {@link Table}. */ public class BasicUnkeyedTable implements UnkeyedTable, HasTableOperations { @@ -318,4 +319,9 @@ public class BasicUnkeyedTable implements UnkeyedTable, HasTableOperations { public UpdatePartitionProperties updatePartitionProperties(Transaction transaction) { return new PartitionPropertiesUpdate(this, transaction); } + + @Override + public UUID uuid() { + return UUID.fromString(this.operations().current().uuid()); + } } diff --git a/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java b/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java index bd674c7d3..e7fa4a37e 100644 --- a/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java +++ b/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java @@ -25,24 +25,44 @@ import org.apache.amoro.shade.guava32.com.google.common.collect.Sets; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; +import org.junit.After; import org.junit.Assert; import org.junit.Assume; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import java.util.Arrays; +import java.util.List; import java.util.Set; @RunWith(Parameterized.class) -public class TestIcebergFindFiles extends TableTestBase { +public class TestIcebergFindFiles extends TestBase { + @Parameterized.Parameters(name = "formatVersion = {0}") - public static Object[] parameters() { - return new Object[] {1, 2}; + public static List<Object> parameters() { + return Arrays.asList(1, 2); } + @Rule public TemporaryFolder tempDir = new TemporaryFolder(); + public TestIcebergFindFiles(int formatVersion) { - super(formatVersion); + this.formatVersion = formatVersion; + } + + @Before + public void setUp() throws Exception { + this.tableDir = tempDir.newFolder(); + this.tableDir.delete(); + super.setupTable(); + } + + @After + public void cleanUp() throws Exception { + super.cleanupTables(); } @Test diff --git a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml index 41a51b9d8..b173af469 100644 --- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml +++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml @@ -34,10 +34,11 @@ <packaging>jar</packaging> <properties> + <iceberg.version>1.4.3</iceberg.version> <kafka.version>3.2.3</kafka.version> <assertj.version>3.21.0</assertj.version> <testcontainers.version>1.17.2</testcontainers.version> - <flink.version>1.17.1</flink.version> + <flink.version>1.17.2</flink.version> </properties> <dependencies> diff --git a/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml b/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml index 0f9202e68..223f8e795 100644 --- a/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml +++ b/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml @@ -34,6 +34,7 @@ <packaging>jar</packaging> <properties> + <iceberg.version>1.4.3</iceberg.version> <kafka.version>2.8.1</kafka.version> <assertj.version>3.21.0</assertj.version> <testcontainers.version>1.17.2</testcontainers.version> diff --git a/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml b/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml index cee742267..9ccfd0a6b 100644 --- a/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml +++ b/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml @@ -34,10 +34,11 @@ <packaging>jar</packaging> <properties> + <iceberg.version>1.4.3</iceberg.version> <kafka.version>3.2.3</kafka.version> <assertj.version>3.21.0</assertj.version> <testcontainers.version>1.17.2</testcontainers.version> - <flink.version>1.16.2</flink.version> + <flink.version>1.16.3</flink.version> </properties> <dependencies> diff --git a/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml b/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml index 5fd713cca..669b76b5b 100644 --- a/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml +++ b/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml @@ -34,10 +34,11 @@ <packaging>jar</packaging> <properties> + <iceberg.version>1.4.3</iceberg.version> <kafka.version>3.2.3</kafka.version> <assertj.version>3.21.0</assertj.version> <testcontainers.version>1.17.2</testcontainers.version> - <flink.version>1.17.1</flink.version> + <flink.version>1.17.2</flink.version> </properties> <dependencies> diff --git a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml index dd4fa0995..e3ca63138 100644 --- a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml +++ b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml @@ -33,6 +33,7 @@ <url>https://amoro.apache.org</url> <properties> + <iceberg.version>1.4.3</iceberg.version> <spark.version>3.2.4</spark.version> <scala.version>2.12.15</scala.version> </properties> @@ -144,7 +145,7 @@ <dependency> <groupId>org.apache.iceberg</groupId> - <artifactId>iceberg-spark-3.2_2.12</artifactId> + <artifactId>iceberg-spark-3.3_2.12</artifactId> <version>${iceberg.version}</version> <scope>provided</scope> <exclusions> diff --git a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml index cc4e7a8d3..e5d5cf667 100644 --- a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml +++ b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml @@ -34,6 +34,7 @@ <url>https://amoro.apache.org</url> <properties> + <iceberg.version>1.4.3</iceberg.version> <hive.version>2.3.9</hive.version> <spark.version>3.2.4</spark.version> <scala.version>2.12.15</scala.version> diff --git a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml index 8d6cbff8b..2dd91e9ba 100644 --- a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml +++ b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml @@ -34,6 +34,7 @@ <url>https://amoro.apache.org</url> <properties> + <iceberg.version>1.4.3</iceberg.version> <hive.version>2.3.9</hive.version> <spark.version>3.3.2</spark.version> <scala.version>2.12.15</scala.version> diff --git a/pom.xml b/pom.xml index 97a882b18..458481f38 100644 --- a/pom.xml +++ b/pom.xml @@ -101,7 +101,7 @@ <maven-clean-plugin.version>3.3.2</maven-clean-plugin.version> <maven-checkstyle-plugin.version>3.3.1</maven-checkstyle-plugin.version> - <iceberg.version>1.4.3</iceberg.version> + <iceberg.version>1.6.1</iceberg.version> <paimon.version>0.9.0</paimon.version> <hive.version>3.1.1</hive.version> <hadoop.version>3.4.0</hadoop.version> @@ -1360,8 +1360,8 @@ <guava.version>14.0.1</guava.version> <hive.version>2.3.9</hive.version> <hadoop.version>2.10.2</hadoop.version> - <terminal.spark.version>3.2.4</terminal.spark.version> - <terminal.spark.major.version>3.2</terminal.spark.major.version> + <terminal.spark.version>3.3.3</terminal.spark.version> + <terminal.spark.major.version>3.3</terminal.spark.major.version> </properties> </profile> <profile>