This is an automated email from the ASF dual-hosted git repository.
yuanzhou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0f5716f91 [GLUTEN-4933][VL] Update iceberg version to 1.4.3 for Spark
3.4 and above (#4967)
0f5716f91 is described below
commit 0f5716f918a18be2ce69c1f84cc9b3e87e843caa
Author: Yan Ma <[email protected]>
AuthorDate: Thu Mar 21 08:19:35 2024 +0800
[GLUTEN-4933][VL] Update iceberg version to 1.4.3 for Spark 3.4 and above
(#4967)
This patch made a clean up on the iceberg used for different Spark shims
Spark 3.2 : v1.3.1
Spark 3.3 : v1.3.1
Spark 3.4 : v1.4.3
Spark 3.5 : v1.4.3
Existing unit tests on spark w/ iceberg passed
---
pom.xml | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git a/pom.xml b/pom.xml
index 54c9fd093..6bdbf678f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -39,7 +39,6 @@
<properties>
<caffeine.version.java8>2.9.3</caffeine.version.java8>
<delta.version>2.0.1</delta.version>
- <iceberg.version>1.3.1</iceberg.version>
<delta.binary.version>20</delta.binary.version>
<scala.binary.version>2.12</scala.binary.version>
<scala.version>2.12.15</scala.version>
@@ -113,7 +112,8 @@
<properties>
<sparkbundle.version>3.2</sparkbundle.version>
<sparkshim.artifactId>spark-sql-columnar-shims-spark32</sparkshim.artifactId>
- <spark.version>3.2.2</spark.version>
+ <spark.version>3.2.2</spark.version>
+ <iceberg.version>1.3.1</iceberg.version>
<delta.version>2.0.1</delta.version>
<delta.binary.version>20</delta.binary.version>
</properties>
@@ -123,8 +123,10 @@
<properties>
<sparkbundle.version>3.3</sparkbundle.version>
<sparkshim.artifactId>spark-sql-columnar-shims-spark33</sparkshim.artifactId>
- <spark.version>3.3.1</spark.version>
- <delta.version>2.2.0</delta.version>
+ <spark.version>3.3.1</spark.version>
+ <!-- keep using iceberg v1.3.1 for parquet compatibilty. -->
+ <iceberg.version>1.3.1</iceberg.version>
+ <delta.version>2.2.0</delta.version>
<delta.binary.version>22</delta.binary.version>
</properties>
</profile>
@@ -133,8 +135,9 @@
<properties>
<sparkbundle.version>3.4</sparkbundle.version>
<sparkshim.artifactId>spark-sql-columnar-shims-spark34</sparkshim.artifactId>
- <spark.version>3.4.2</spark.version>
- <delta.version>2.4.0</delta.version>
+ <spark.version>3.4.2</spark.version>
+ <iceberg.version>1.4.3</iceberg.version>
+ <delta.version>2.4.0</delta.version>
<delta.binary.version>24</delta.binary.version>
</properties>
</profile>
@@ -143,7 +146,8 @@
<properties>
<sparkbundle.version>3.5</sparkbundle.version>
<sparkshim.artifactId>spark-sql-columnar-shims-spark35</sparkshim.artifactId>
- <spark.version>3.5.1</spark.version>
+ <spark.version>3.5.1</spark.version>
+ <iceberg.version>1.4.3</iceberg.version>
<delta.version>2.4.0</delta.version>
<delta.binary.version>24</delta.binary.version>
</properties>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
