This is an automated email from the ASF dual-hosted git repository.

jinsongzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git


The following commit(s) were added to refs/heads/master by this push:
     new 957e52124 [AMORO-3084] Upgrade to iceberg 1.6.1 (#3071)
957e52124 is described below

commit 957e52124af7c7791af4e7196f421311fc55ceeb
Author: ConradJam <jam.gz...@gmail.com>
AuthorDate: Thu Jan 16 17:00:19 2025 +0800

    [AMORO-3084] Upgrade to iceberg 1.6.1 (#3071)
    
    [Improvement]: Upgrade to iceberg 1.6.1 (#3084)
---
 .../amoro/optimizing/IcebergRewriteExecutor.java   | 13 ----------
 .../org/apache/amoro/table/BasicUnkeyedTable.java  |  6 +++++
 .../org/apache/iceberg/TestIcebergFindFiles.java   | 28 ++++++++++++++++++----
 .../amoro-mixed-flink-common/pom.xml               |  3 ++-
 .../v1.15/amoro-mixed-flink-1.15/pom.xml           |  1 +
 .../v1.16/amoro-mixed-flink-1.16/pom.xml           |  3 ++-
 .../v1.17/amoro-mixed-flink-1.17/pom.xml           |  3 ++-
 .../amoro-mixed-spark-3-common/pom.xml             |  3 ++-
 .../v3.2/amoro-mixed-spark-3.2/pom.xml             |  1 +
 .../v3.3/amoro-mixed-spark-3.3/pom.xml             |  1 +
 pom.xml                                            |  6 ++---
 11 files changed, 44 insertions(+), 24 deletions(-)

diff --git 
a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java
 
b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java
index fa355a8bf..d17222172 100644
--- 
a/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java
+++ 
b/amoro-format-iceberg/src/main/java/org/apache/amoro/optimizing/IcebergRewriteExecutor.java
@@ -24,7 +24,6 @@ import 
org.apache.amoro.io.writer.IcebergFanoutPosDeleteWriter;
 import org.apache.amoro.table.MixedTable;
 import org.apache.amoro.utils.map.StructLikeCollections;
 import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.StructLike;
 import org.apache.iceberg.TableProperties;
 import org.apache.iceberg.data.GenericAppenderFactory;
 import org.apache.iceberg.data.IdentityPartitionConverters;
@@ -46,18 +45,6 @@ public class IcebergRewriteExecutor extends 
AbstractRewriteFilesExecutor {
     super(input, table, structLikeCollections);
   }
 
-  // TODO We can remove this override method after upgrading Iceberg version 
to 1.5+.
-  @Override
-  protected StructLike partition() {
-    StructLike partitionData = super.partition();
-    if (partitionData != null && partitionData.size() == 0) {
-      // Cast empty partition data to NULL to avoid creating empty partition 
directory.
-      return null;
-    } else {
-      return partitionData;
-    }
-  }
-
   @Override
   protected OptimizingDataReader dataReader() {
     return new GenericCombinedIcebergDataReader(
diff --git 
a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
 
b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
index d5f6f878d..6c5cceee6 100644
--- 
a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
+++ 
b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/BasicUnkeyedTable.java
@@ -65,6 +65,7 @@ import org.apache.iceberg.util.StructLikeMap;
 
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 
 /** Basic implementation of {@link UnkeyedTable}, wrapping a {@link Table}. */
 public class BasicUnkeyedTable implements UnkeyedTable, HasTableOperations {
@@ -318,4 +319,9 @@ public class BasicUnkeyedTable implements UnkeyedTable, 
HasTableOperations {
   public UpdatePartitionProperties updatePartitionProperties(Transaction 
transaction) {
     return new PartitionPropertiesUpdate(this, transaction);
   }
+
+  @Override
+  public UUID uuid() {
+    return UUID.fromString(this.operations().current().uuid());
+  }
 }
diff --git 
a/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
 
b/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
index bd674c7d3..e7fa4a37e 100644
--- 
a/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
+++ 
b/amoro-format-iceberg/src/test/java/org/apache/iceberg/TestIcebergFindFiles.java
@@ -25,24 +25,44 @@ import 
org.apache.amoro.shade.guava32.com.google.common.collect.Sets;
 import org.apache.iceberg.expressions.Expressions;
 import org.apache.iceberg.types.Conversions;
 import org.apache.iceberg.types.Types;
+import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
 import java.util.Arrays;
+import java.util.List;
 import java.util.Set;
 
 @RunWith(Parameterized.class)
-public class TestIcebergFindFiles extends TableTestBase {
+public class TestIcebergFindFiles extends TestBase {
+
   @Parameterized.Parameters(name = "formatVersion = {0}")
-  public static Object[] parameters() {
-    return new Object[] {1, 2};
+  public static List<Object> parameters() {
+    return Arrays.asList(1, 2);
   }
 
+  @Rule public TemporaryFolder tempDir = new TemporaryFolder();
+
   public TestIcebergFindFiles(int formatVersion) {
-    super(formatVersion);
+    this.formatVersion = formatVersion;
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    this.tableDir = tempDir.newFolder();
+    this.tableDir.delete();
+    super.setupTable();
+  }
+
+  @After
+  public void cleanUp() throws Exception {
+    super.cleanupTables();
   }
 
   @Test
diff --git 
a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml 
b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml
index 41a51b9d8..b173af469 100644
--- a/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-flink/amoro-mixed-flink-common/pom.xml
@@ -34,10 +34,11 @@
     <packaging>jar</packaging>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <kafka.version>3.2.3</kafka.version>
         <assertj.version>3.21.0</assertj.version>
         <testcontainers.version>1.17.2</testcontainers.version>
-        <flink.version>1.17.1</flink.version>
+        <flink.version>1.17.2</flink.version>
     </properties>
 
     <dependencies>
diff --git 
a/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml 
b/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml
index 0f9202e68..223f8e795 100644
--- a/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-flink/v1.15/amoro-mixed-flink-1.15/pom.xml
@@ -34,6 +34,7 @@
     <packaging>jar</packaging>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <kafka.version>2.8.1</kafka.version>
         <assertj.version>3.21.0</assertj.version>
         <testcontainers.version>1.17.2</testcontainers.version>
diff --git 
a/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml 
b/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml
index cee742267..9ccfd0a6b 100644
--- a/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-flink/v1.16/amoro-mixed-flink-1.16/pom.xml
@@ -34,10 +34,11 @@
     <packaging>jar</packaging>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <kafka.version>3.2.3</kafka.version>
         <assertj.version>3.21.0</assertj.version>
         <testcontainers.version>1.17.2</testcontainers.version>
-        <flink.version>1.16.2</flink.version>
+        <flink.version>1.16.3</flink.version>
     </properties>
 
     <dependencies>
diff --git 
a/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml 
b/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
index 5fd713cca..669b76b5b 100644
--- a/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-flink/v1.17/amoro-mixed-flink-1.17/pom.xml
@@ -34,10 +34,11 @@
     <packaging>jar</packaging>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <kafka.version>3.2.3</kafka.version>
         <assertj.version>3.21.0</assertj.version>
         <testcontainers.version>1.17.2</testcontainers.version>
-        <flink.version>1.17.1</flink.version>
+        <flink.version>1.17.2</flink.version>
     </properties>
 
     <dependencies>
diff --git 
a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml 
b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
index dd4fa0995..e3ca63138 100644
--- a/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/amoro-mixed-spark-3-common/pom.xml
@@ -33,6 +33,7 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <spark.version>3.2.4</spark.version>
         <scala.version>2.12.15</scala.version>
     </properties>
@@ -144,7 +145,7 @@
 
         <dependency>
             <groupId>org.apache.iceberg</groupId>
-            <artifactId>iceberg-spark-3.2_2.12</artifactId>
+            <artifactId>iceberg-spark-3.3_2.12</artifactId>
             <version>${iceberg.version}</version>
             <scope>provided</scope>
             <exclusions>
diff --git 
a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml 
b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
index cc4e7a8d3..e5d5cf667 100644
--- a/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/v3.2/amoro-mixed-spark-3.2/pom.xml
@@ -34,6 +34,7 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <hive.version>2.3.9</hive.version>
         <spark.version>3.2.4</spark.version>
         <scala.version>2.12.15</scala.version>
diff --git 
a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml 
b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
index 8d6cbff8b..2dd91e9ba 100644
--- a/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
+++ b/amoro-format-mixed/amoro-mixed-spark/v3.3/amoro-mixed-spark-3.3/pom.xml
@@ -34,6 +34,7 @@
     <url>https://amoro.apache.org</url>
 
     <properties>
+        <iceberg.version>1.4.3</iceberg.version>
         <hive.version>2.3.9</hive.version>
         <spark.version>3.3.2</spark.version>
         <scala.version>2.12.15</scala.version>
diff --git a/pom.xml b/pom.xml
index 97a882b18..458481f38 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@
         <maven-clean-plugin.version>3.3.2</maven-clean-plugin.version>
         
<maven-checkstyle-plugin.version>3.3.1</maven-checkstyle-plugin.version>
 
-        <iceberg.version>1.4.3</iceberg.version>
+        <iceberg.version>1.6.1</iceberg.version>
         <paimon.version>0.9.0</paimon.version>
         <hive.version>3.1.1</hive.version>
         <hadoop.version>3.4.0</hadoop.version>
@@ -1360,8 +1360,8 @@
                 <guava.version>14.0.1</guava.version>
                 <hive.version>2.3.9</hive.version>
                 <hadoop.version>2.10.2</hadoop.version>
-                <terminal.spark.version>3.2.4</terminal.spark.version>
-                
<terminal.spark.major.version>3.2</terminal.spark.major.version>
+                <terminal.spark.version>3.3.3</terminal.spark.version>
+                
<terminal.spark.major.version>3.3</terminal.spark.major.version>
             </properties>
         </profile>
         <profile>

Reply via email to