This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 3a06cc363 [KYUUBI #5202] Switch to Spark 3.4 by default
3a06cc363 is described below
commit 3a06cc3637875b7d8f07ae3d37899525edde2a08
Author: Cheng Pan <[email protected]>
AuthorDate: Sun Aug 27 23:17:10 2023 +0800
[KYUUBI #5202] Switch to Spark 3.4 by default
### _Why are the changes needed?_
Kyuubi fully supports Spark 3.4 now, it's time to move forward.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including
negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run
test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests)
locally before make a pull request
### _Was this patch authored or co-authored using generative AI tooling?_
Closes #5202 from pan3793/default-3.4.
Closes #5202
a0efccdbf [Cheng Pan] nit
30456dbb9 [Cheng Pan] nit
1cc83c871 [Cheng Pan] enable lineage test
d8ca7c7d8 [Cheng Pan] Switch to Spark 3.4 by default
Authored-by: Cheng Pan <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
---
.github/workflows/master.yml | 16 ++++++----------
.../test/deployment/KyuubiOnKubernetesTestsSuite.scala | 2 +-
.../test/spark/SparkOnKubernetesTestsSuite.scala | 2 +-
pom.xml | 11 +++++++----
4 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 0e050b4a7..b2dd57a50 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -57,20 +57,20 @@ jobs:
comment: ["normal"]
include:
- java: 8
- spark: '3.3'
+ spark: '3.4'
spark-archive:
'-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3
-Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz -Pzookeeper-3.6'
exclude-tags:
'-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-3.1-binary'
- java: 8
- spark: '3.3'
+ spark: '3.4'
spark-archive:
'-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4
-Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6'
exclude-tags:
'-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
comment: 'verify-on-spark-3.2-binary'
- java: 8
- spark: '3.3'
- spark-archive:
'-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.4.0
-Dspark.archive.name=spark-3.4.0-bin-hadoop3.tgz -Pzookeeper-3.6'
+ spark: '3.4'
+ spark-archive:
'-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3
-Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6'
exclude-tags:
'-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
- comment: 'verify-on-spark-3.4-binary'
+ comment: 'verify-on-spark-3.3-binary'
exclude:
# SPARK-33772: Spark supports JDK 17 since 3.3.0
- java: 17
@@ -101,16 +101,12 @@ jobs:
- name: Build and test Kyuubi and Spark with maven w/o linters
run: |
TEST_MODULES="dev/kyuubi-codecov"
- if [[ "${{ matrix.spark }}" == "3.4" ]]; then
- # FIXME: Spark 3.4 supports lineage plugin
- TEST_MODULES="$TEST_MODULES,!extensions/spark/kyuubi-spark-lineage"
- fi
./build/mvn clean install ${MVN_OPT} -pl ${TEST_MODULES} -am \
-Pspark-${{ matrix.spark }} ${{ matrix.spark-archive }} ${{
matrix.exclude-tags }}
- name: Code coverage
if: |
matrix.java == 8 &&
- matrix.spark == '3.2' &&
+ matrix.spark == '3.4' &&
matrix.spark-archive == ''
uses: codecov/codecov-action@v3
with:
diff --git
a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
index d57ef3742..73cb5620a 100644
---
a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
+++
b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
@@ -54,7 +54,7 @@ class KyuubiOnKubernetesWithSparkTestsBase extends
WithKyuubiServerOnKubernetes
super.connectionConf ++
Map(
"spark.master" -> s"k8s://$miniKubeApiMaster",
- "spark.kubernetes.container.image" -> "apache/spark:3.3.3",
+ "spark.kubernetes.container.image" -> "apache/spark:3.4.1",
"spark.executor.memory" -> "512M",
"spark.driver.memory" -> "1024M",
"spark.kubernetes.driver.request.cores" -> "250m",
diff --git
a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
index ba68d2090..037681a3f 100644
---
a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
+++
b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala
@@ -51,7 +51,7 @@ abstract class SparkOnKubernetesSuiteBase
// TODO Support more Spark version
// Spark official docker image: https://hub.docker.com/r/apache/spark/tags
KyuubiConf().set("spark.master", s"k8s://$apiServerAddress")
- .set("spark.kubernetes.container.image", "apache/spark:3.3.3")
+ .set("spark.kubernetes.container.image", "apache/spark:3.4.1")
.set("spark.kubernetes.container.image.pullPolicy", "IfNotPresent")
.set("spark.executor.instances", "1")
.set("spark.executor.memory", "512M")
diff --git a/pom.xml b/pom.xml
index d90ef6430..36edcee40 100644
--- a/pom.xml
+++ b/pom.xml
@@ -133,7 +133,7 @@
<commons-lang.version>2.6</commons-lang.version>
<commons-lang3.version>3.12.0</commons-lang3.version>
<etcd.version>0.7.3</etcd.version>
- <delta.version>2.3.0</delta.version>
+ <delta.version>2.4.0</delta.version>
<failsafe.verion>2.4.4</failsafe.verion>
<fb303.version>0.9.3</fb303.version>
<flexmark.version>0.62.2</flexmark.version>
@@ -196,8 +196,8 @@
DO NOT forget to change the following properties when change the
minor version of Spark:
`delta.version`, `maven.plugin.scalatest.exclude.tags`
-->
- <spark.version>3.3.3</spark.version>
- <spark.binary.version>3.3</spark.binary.version>
+ <spark.version>3.4.1</spark.version>
+ <spark.binary.version>3.4</spark.binary.version>
<spark.archive.name>spark-${spark.version}-bin-hadoop3.tgz</spark.archive.name>
<spark.archive.mirror>${apache.archive.dist}/spark/spark-${spark.version}</spark.archive.mirror>
<spark.archive.download.skip>false</spark.archive.download.skip>
@@ -233,7 +233,7 @@
<maven.plugin.frontend.version>1.12.1</maven.plugin.frontend.version>
<maven.plugin.scala.version>4.8.0</maven.plugin.scala.version>
<maven.plugin.scalatest.version>2.2.0</maven.plugin.scalatest.version>
-
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow</maven.plugin.scalatest.exclude.tags>
+
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.IcebergTest</maven.plugin.scalatest.exclude.tags>
<maven.plugin.scalatest.include.tags></maven.plugin.scalatest.include.tags>
<maven.plugin.scalatest.debug.enabled>false</maven.plugin.scalatest.debug.enabled>
<maven.plugin.spotless.version>2.30.0</maven.plugin.spotless.version>
@@ -2213,6 +2213,9 @@
<module>extensions/spark/kyuubi-spark-connector-hive</module>
</modules>
<properties>
+ <delta.version>2.3.0</delta.version>
+ <spark.version>3.3.3</spark.version>
+ <spark.binary.version>3.3</spark.binary.version>
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow</maven.plugin.scalatest.exclude.tags>
</properties>
</profile>
