This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new a07c57f06 [KYUUBI #6427] Extract data lake artifact names as maven
properties
a07c57f06 is described below
commit a07c57f0644dec3aa5a7265560852c3b33fd387b
Author: Cheng Pan <[email protected]>
AuthorDate: Wed Jun 5 15:23:45 2024 +0800
[KYUUBI #6427] Extract data lake artifact names as maven properties
# :mag: Description
Improve data lake dependency management by extracting the following Maven
properties:
- `delta.artifact`
- `hudi.artifact`
- `iceberg.artifact`
- `paimon.artifact`
It often takes a while for the downstream data lakes to support the new
Spark versions, extracting those properties makes it easy to override in the
new profile on the Kyuubi project's `pom.xml` to workaround before data lakes
jars are available.
One use case is
https://github.com/apache/kyuubi/pull/6407/commits/a19bb7c18e8e22181b7cebb1e4166868bd5525fb
## Types of changes :bookmark:
- [ ] Bugfix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
## Test Plan ๐งช
Pass GHA.
---
# Checklist ๐
- [x] This patch was not authored or co-authored using [Generative
Tooling](https://www.apache.org/legal/generative-tooling.html)
**Be nice. Be informative.**
Closes #6427 from pan3793/datalake-dep.
Closes #6427
74a9300e0 [Cheng Pan] Improve datalake dependency management
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
extensions/spark/kyuubi-spark-authz/pom.xml | 8 ++++----
externals/kyuubi-spark-sql-engine/pom.xml | 4 ++--
kyuubi-server/pom.xml | 4 ++--
pom.xml | 28 +++++++++++++++-------------
4 files changed, 23 insertions(+), 21 deletions(-)
diff --git a/extensions/spark/kyuubi-spark-authz/pom.xml
b/extensions/spark/kyuubi-spark-authz/pom.xml
index c3fde22ec..f420b3386 100644
--- a/extensions/spark/kyuubi-spark-authz/pom.xml
+++ b/extensions/spark/kyuubi-spark-authz/pom.xml
@@ -359,7 +359,7 @@
<dependency>
<groupId>org.apache.iceberg</groupId>
-
<artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
+ <artifactId>${iceberg.artifact}</artifactId>
<scope>test</scope>
</dependency>
@@ -371,19 +371,19 @@
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark-${spark.binary.version}</artifactId>
+ <artifactId>${paimon.artifact}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.delta</groupId>
- <artifactId>${delta.artifact}_${scala.binary.version}</artifactId>
+ <artifactId>${delta.artifact}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
-
<artifactId>hudi-spark${hudi.spark.binary.version}-bundle_${scala.binary.version}</artifactId>
+ <artifactId>${hudi.artifact}</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/externals/kyuubi-spark-sql-engine/pom.xml
b/externals/kyuubi-spark-sql-engine/pom.xml
index 53510c6ba..9dd0f9e09 100644
--- a/externals/kyuubi-spark-sql-engine/pom.xml
+++ b/externals/kyuubi-spark-sql-engine/pom.xml
@@ -155,13 +155,13 @@
<dependency>
<groupId>org.apache.iceberg</groupId>
-
<artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
+ <artifactId>${iceberg.artifact}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.delta</groupId>
- <artifactId>${delta.artifact}_${scala.binary.version}</artifactId>
+ <artifactId>${delta.artifact}</artifactId>
<scope>test</scope>
</dependency>
diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml
index a2bdf9e79..314393435 100644
--- a/kyuubi-server/pom.xml
+++ b/kyuubi-server/pom.xml
@@ -314,13 +314,13 @@
<dependency>
<groupId>org.apache.iceberg</groupId>
-
<artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
+ <artifactId>${iceberg.artifact}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.delta</groupId>
- <artifactId>${delta.artifact}_${scala.binary.version}</artifactId>
+ <artifactId>${delta.artifact}</artifactId>
<scope>test</scope>
</dependency>
diff --git a/pom.xml b/pom.xml
index 576020f23..cc0460304 100644
--- a/pom.xml
+++ b/pom.xml
@@ -134,7 +134,7 @@
<commons-io.version>2.11.0</commons-io.version>
<commons-lang.version>2.6</commons-lang.version>
<commons-lang3.version>3.13.0</commons-lang3.version>
- <delta.artifact>delta-spark</delta.artifact>
+ <delta.artifact>delta-spark_${scala.binary.version}</delta.artifact>
<delta.version>3.2.0</delta.version>
<failsafe.verion>3.3.2</failsafe.verion>
<fb303.version>0.9.3</fb303.version>
@@ -157,8 +157,9 @@
<httpclient.version>4.5.14</httpclient.version>
<httpcore.version>4.4.16</httpcore.version>
<hudi.version>0.15.0</hudi.version>
-
<hudi.spark.binary.version>${spark.binary.version}</hudi.spark.binary.version>
+
<hudi.artifact>hudi-spark${spark.binary.version}-bundle_${scala.binary.version}</hudi.artifact>
<iceberg.version>1.5.2</iceberg.version>
+
<iceberg.artifact>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</iceberg.artifact>
<jackson.version>2.15.4</jackson.version>
<javax.servlet-api.version>4.0.1</javax.servlet-api.version>
<!-- 6.0.0 requires JDK 11 -->
@@ -182,6 +183,7 @@
<openai.java.version>0.12.0</openai.java.version>
<retrofit.version>2.9.0</retrofit.version>
<paimon.version>0.7.0-incubating</paimon.version>
+ <paimon.artifact>paimon-spark-${spark.binary.version}</paimon.artifact>
<phoenix.version>6.0.0</phoenix.version>
<postgresql.version>42.7.2</postgresql.version>
<prometheus.version>0.16.0</prometheus.version>
@@ -951,13 +953,13 @@
<dependency>
<groupId>org.apache.iceberg</groupId>
-
<artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
+ <artifactId>${iceberg.artifact}</artifactId>
<version>${iceberg.version}</version>
</dependency>
<dependency>
<groupId>io.delta</groupId>
-
<artifactId>${delta.artifact}_${scala.binary.version}</artifactId>
+ <artifactId>${delta.artifact}</artifactId>
<version>${delta.version}</version>
<exclusions>
<!--
@@ -1185,13 +1187,13 @@
<dependency>
<groupId>org.apache.hudi</groupId>
-
<artifactId>hudi-spark${hudi.spark.binary.version}-bundle_${scala.binary.version}</artifactId>
+ <artifactId>${hudi.artifact}</artifactId>
<version>${hudi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark-${spark.binary.version}</artifactId>
+ <artifactId>${paimon.artifact}</artifactId>
<version>${paimon.version}</version>
</dependency>
</dependencies>
@@ -1979,7 +1981,7 @@
<properties>
<spark.version>3.2.4</spark.version>
<spark.binary.version>3.2</spark.binary.version>
- <delta.artifact>delta-core</delta.artifact>
+
<delta.artifact>delta-core_${scala.binary.version}</delta.artifact>
<delta.version>2.0.2</delta.version>
<!-- Iceberg 1.5.0 removed support for Spark 3.2.
apache/iceberg#9295 -->
<iceberg.version>1.4.3</iceberg.version>
@@ -1996,10 +1998,10 @@
<module>extensions/spark/kyuubi-spark-connector-hive</module>
</modules>
<properties>
- <delta.version>2.3.0</delta.version>
- <delta.artifact>delta-core</delta.artifact>
<spark.version>3.3.4</spark.version>
<spark.binary.version>3.3</spark.binary.version>
+ <delta.version>2.3.0</delta.version>
+
<delta.artifact>delta-core_${scala.binary.version}</delta.artifact>
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow</maven.plugin.scalatest.exclude.tags>
</properties>
</profile>
@@ -2011,10 +2013,10 @@
<module>extensions/spark/kyuubi-spark-connector-hive</module>
</modules>
<properties>
- <delta.artifact>delta-core</delta.artifact>
- <delta.version>2.4.0</delta.version>
<spark.version>3.4.3</spark.version>
<spark.binary.version>3.4</spark.binary.version>
+ <delta.version>2.4.0</delta.version>
+
<delta.artifact>delta-core_${scala.binary.version}</delta.artifact>
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow</maven.plugin.scalatest.exclude.tags>
</properties>
</profile>
@@ -2026,10 +2028,10 @@
<module>extensions/spark/kyuubi-spark-connector-hive</module>
</modules>
<properties>
- <delta.artifact>delta-spark</delta.artifact>
- <delta.version>3.2.0</delta.version>
<spark.version>3.5.1</spark.version>
<spark.binary.version>3.5</spark.binary.version>
+ <delta.version>3.2.0</delta.version>
+
<delta.artifact>delta-spark_${scala.binary.version}</delta.artifact>
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow</maven.plugin.scalatest.exclude.tags>
</properties>
</profile>
