This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 4be4864082d7 chore(docker): move demo/IT images to Spark 3.5.9 (#19888)
4be4864082d7 is described below
commit 4be4864082d7d787d70dec6cf2a91f7f04866eb8
Author: voonhous <[email protected]>
AuthorDate: Thu Sep 10 16:50:11 2026 +0800
chore(docker): move demo/IT images to Spark 3.5.9 (#19888)
The docker demo stack and the docker-cluster ITs were still on
Spark 3.5.3 while the root pom moves to 3.5.9 (#19884), so the
containers ran a bundle compiled against a newer Spark than their
runtime. Move every pin in one go: the four Spark Dockerfiles,
docker.spark.version, the build_docker_images.sh default, both
compose sets (renamed spark353 -> spark359), setup/stop_demo.sh,
hudi-integ-test's dockerCompose.file, the README, and the parked
integration-tests-hive-sync matrix.
The 3.5.9 images do not exist on Docker Hub yet. integration-tests
runs docker compose up on the hadoop284 file every run, so this
must not merge until they are pushed; #19887 has the build and
push steps.
---
.github/workflows/bot.yml | 6 +++---
docker/README.md | 10 +++++-----
docker/build_docker_images.sh | 2 +-
...> docker-compose_hadoop284_hive2310_spark359_amd64.yml} | 14 +++++++-------
...> docker-compose_hadoop284_hive2310_spark359_arm64.yml} | 14 +++++++-------
...=> docker-compose_hadoop334_hive313_spark359_amd64.yml} | 14 +++++++-------
...=> docker-compose_hadoop334_hive313_spark359_arm64.yml} | 14 +++++++-------
docker/hoodie/hadoop/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/Dockerfile | 2 +-
docker/hoodie/hadoop/sparkadhoc/Dockerfile | 2 +-
docker/hoodie/hadoop/sparkmaster/Dockerfile | 2 +-
docker/hoodie/hadoop/sparkworker/Dockerfile | 2 +-
docker/setup_demo.sh | 4 ++--
docker/stop_demo.sh | 4 ++--
hudi-integ-test/pom.xml | 4 ++--
15 files changed, 48 insertions(+), 48 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index a8cc61a87c63..ae3deffc293e 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -1562,7 +1562,7 @@ jobs:
# integration-tests-hive-sync:
# # Testcontainers-based E2E hive sync coverage for Hudi's custom logical
types
# # (VECTOR, BLOB) and the Spark 4.0+ VARIANT type. Runs the integ2
testcontainers
-# # suite (ITTestCustomTypeHiveSync) against a real Hive metastore on Spark
3.5.3,
+# # suite (ITTestCustomTypeHiveSync) against a real Hive metastore on Spark
3.5.9,
# # 4.0.2, and 4.1.1 stacks.
# runs-on: ubuntu-latest
# needs: changes
@@ -1574,8 +1574,8 @@ jobs:
# scalaProfile: '-Dscala-2.12 -Dscala.binary.version=2.12'
# flinkProfile: 'flink1.20'
# jdkVersion: '11'
-# composePrefix: 'docker-compose_hadoop284_hive2310_spark353'
-# sparkAdhocImage:
'apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.3:latest'
+# composePrefix: 'docker-compose_hadoop284_hive2310_spark359'
+# sparkAdhocImage:
'apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.9:latest'
# - sparkProfile: 'spark4.0'
# scalaProfile: '-Dscala-2.13 -Dscala.binary.version=2.13'
# flinkProfile: 'flink1.20'
diff --git a/docker/README.md b/docker/README.md
index 5b19e85c4334..a3e6aa1fc8ed 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -70,8 +70,8 @@ To build the Docker demo images with `docker` directly,
rather than through the
`build_local_docker_images.sh` above, run the script from under
`<HUDI_REPO_DIR>/docker`:
```shell
-# With no flags, builds Hadoop 2.8.4 / Spark 3.5.3 / Hive 2.3.10, matching
-# docker-compose_hadoop284_hive2310_spark353_{amd64,arm64}.yml
+# With no flags, builds Hadoop 2.8.4 / Spark 3.5.9 / Hive 2.3.10, matching
+# docker-compose_hadoop284_hive2310_spark359_{amd64,arm64}.yml
./build_docker_images.sh
```
@@ -81,15 +81,15 @@ combinations under `docker/compose`.
```shell
# Matches setup_demo.sh and
-# docker-compose_hadoop334_hive313_spark353_{amd64,arm64}.yml
-./build_docker_images.sh --hadoop-version 3.3.4 --spark-version 3.5.3
--hive-version 3.1.3
+# docker-compose_hadoop334_hive313_spark359_{amd64,arm64}.yml
+./build_docker_images.sh --hadoop-version 3.3.4 --spark-version 3.5.9
--hive-version 3.1.3
# Another supported combination is
# docker-compose_hadoop340_hive313_spark401_{amd64,arm64}.yml
./build_docker_images.sh --hadoop-version 3.4.0 --spark-version 4.0.1
--hive-version 3.1.3
```
-`setup_demo.sh` currently defaults to
`docker-compose_hadoop334_hive313_spark353_{amd64,arm64}.yml`. If you build a
+`setup_demo.sh` currently defaults to
`docker-compose_hadoop334_hive313_spark359_{amd64,arm64}.yml`. If you build a
different image set for the demo flow, update `COMPOSE_FILE_NAME` in
`setup_demo.sh` to point to the matching compose
file before running the script. Run `./setup_demo.sh dev` to use your locally
built images; a plain run pulls the
Docker Hub images over them.
diff --git a/docker/build_docker_images.sh b/docker/build_docker_images.sh
index e68ce44080c9..3e37123e8036 100755
--- a/docker/build_docker_images.sh
+++ b/docker/build_docker_images.sh
@@ -21,7 +21,7 @@
# Default versions for Hadoop, Spark, and Hive
HADOOP_VERSION="2.8.4"
-SPARK_VERSION="3.5.3"
+SPARK_VERSION="3.5.9"
HIVE_VERSION="2.3.10"
VERSION_TAG_ARG="" # Initialize to empty, will be set by command-line arg if
provided
MULTI_ARCH=false
diff --git
a/docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml
b/docker/compose/docker-compose_hadoop284_hive2310_spark359_amd64.yml
similarity index 96%
rename from docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml
rename to docker/compose/docker-compose_hadoop284_hive2310_spark359_amd64.yml
index c302a3c95cca..4089c0dff2b9 100644
--- a/docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml
+++ b/docker/compose/docker-compose_hadoop284_hive2310_spark359_amd64.yml
@@ -21,7 +21,7 @@ services:
hostname: namenode
container_name: namenode
environment:
- - CLUSTER_NAME=hudi_hadoop284_hive2310_spark353
+ - CLUSTER_NAME=hudi_hadoop284_hive2310_spark359
ports:
- "50070:50070"
- "8020:8020"
@@ -39,7 +39,7 @@ services:
container_name: datanode1
hostname: datanode1
environment:
- - CLUSTER_NAME=hudi_hadoop284_hive2310_spark353
+ - CLUSTER_NAME=hudi_hadoop284_hive2310_spark359
env_file:
- ./hadoop.env
ports:
@@ -62,7 +62,7 @@ services:
hostname: historyserver
container_name: historyserver
environment:
- - CLUSTER_NAME=hudi_hadoop284_hive2310_spark353
+ - CLUSTER_NAME=hudi_hadoop284_hive2310_spark359
depends_on:
- "namenode"
links:
@@ -131,7 +131,7 @@ services:
- ${HUDI_WS}:/var/hoodie/ws
sparkmaster:
- image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkmaster_3.5.3:latest
+ image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkmaster_3.5.9:latest
platform: linux/amd64
hostname: sparkmaster
container_name: sparkmaster
@@ -149,7 +149,7 @@ services:
- "namenode"
spark-worker-1:
- image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkworker_3.5.3:latest
+ image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkworker_3.5.9:latest
platform: linux/amd64
hostname: spark-worker-1
container_name: spark-worker-1
@@ -196,7 +196,7 @@ services:
- 8126:8126
adhoc-1:
- image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.3:latest
+ image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.9:latest
platform: linux/amd64
hostname: adhoc-1
container_name: adhoc-1
@@ -219,7 +219,7 @@ services:
- ${HUDI_WS}:/var/hoodie/ws
adhoc-2:
- image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.3:latest
+ image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.9:latest
platform: linux/amd64
hostname: adhoc-2
container_name: adhoc-2
diff --git
a/docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml
b/docker/compose/docker-compose_hadoop284_hive2310_spark359_arm64.yml
similarity index 96%
rename from docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml
rename to docker/compose/docker-compose_hadoop284_hive2310_spark359_arm64.yml
index a9f042155d30..26cac08c7816 100644
--- a/docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml
+++ b/docker/compose/docker-compose_hadoop284_hive2310_spark359_arm64.yml
@@ -20,7 +20,7 @@ services:
hostname: namenode
container_name: namenode
environment:
- - CLUSTER_NAME=hudi_hadoop284_hive2310_spark353
+ - CLUSTER_NAME=hudi_hadoop284_hive2310_spark359
ports:
- "50070:50070"
- "8020:8020"
@@ -37,7 +37,7 @@ services:
container_name: datanode1
hostname: datanode1
environment:
- - CLUSTER_NAME=hudi_hadoop284_hive2310_spark353
+ - CLUSTER_NAME=hudi_hadoop284_hive2310_spark359
env_file:
- ./hadoop.env
ports:
@@ -59,7 +59,7 @@ services:
hostname: historyserver
container_name: historyserver
environment:
- - CLUSTER_NAME=hudi_hadoop284_hive2310_spark353
+ - CLUSTER_NAME=hudi_hadoop284_hive2310_spark359
depends_on:
- "namenode"
links:
@@ -126,7 +126,7 @@ services:
- ${HUDI_WS}:/var/hoodie/ws
sparkmaster:
- image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkmaster_3.5.3:latest
+ image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkmaster_3.5.9:latest
hostname: sparkmaster
container_name: sparkmaster
env_file:
@@ -143,7 +143,7 @@ services:
- "namenode"
spark-worker-1:
- image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkworker_3.5.3:latest
+ image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkworker_3.5.9:latest
hostname: spark-worker-1
container_name: spark-worker-1
env_file:
@@ -189,7 +189,7 @@ services:
- 8126:8126
adhoc-1:
- image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.3:latest
+ image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.9:latest
hostname: adhoc-1
container_name: adhoc-1
env_file:
@@ -211,7 +211,7 @@ services:
- ${HUDI_WS}:/var/hoodie/ws
adhoc-2:
- image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.3:latest
+ image: apachehudi/hudi-hadoop_2.8.4-hive_2.3.10-sparkadhoc_3.5.9:latest
hostname: adhoc-2
container_name: adhoc-2
env_file:
diff --git a/docker/compose/docker-compose_hadoop334_hive313_spark353_amd64.yml
b/docker/compose/docker-compose_hadoop334_hive313_spark359_amd64.yml
similarity index 97%
rename from docker/compose/docker-compose_hadoop334_hive313_spark353_amd64.yml
rename to docker/compose/docker-compose_hadoop334_hive313_spark359_amd64.yml
index d6a054691497..ef3580be0d4e 100644
--- a/docker/compose/docker-compose_hadoop334_hive313_spark353_amd64.yml
+++ b/docker/compose/docker-compose_hadoop334_hive313_spark359_amd64.yml
@@ -19,7 +19,7 @@ services:
hostname: namenode
container_name: namenode
environment:
- - CLUSTER_NAME=hudi_hadoop334_hive313_spark353
+ - CLUSTER_NAME=hudi_hadoop334_hive313_spark359
ports:
- "8020:8020" # HDFS NameNode IPC
- "9000:9000" # HDFS NameNode Client
@@ -37,7 +37,7 @@ services:
container_name: datanode1
hostname: datanode1
environment:
- - CLUSTER_NAME=hudi_hadoop334_hive313_spark353
+ - CLUSTER_NAME=hudi_hadoop334_hive313_spark359
env_file:
- ./hadoop.env
ports:
@@ -60,7 +60,7 @@ services:
hostname: historyserver
container_name: historyserver
environment:
- - CLUSTER_NAME=hudi_hadoop334_hive313_spark353
+ - CLUSTER_NAME=hudi_hadoop334_hive313_spark359
depends_on:
- "namenode"
links:
@@ -160,7 +160,7 @@ services:
- KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
- KAFKA_NUM_PARTITIONS=3
sparkmaster:
- image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkmaster_3.5.3:latest
+ image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkmaster_3.5.9:latest
platform: linux/amd64
hostname: sparkmaster
container_name: sparkmaster
@@ -181,7 +181,7 @@ services:
- "hive-metastore-postgresql"
- "namenode"
spark-worker-1:
- image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkworker_3.5.3:latest
+ image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkworker_3.5.9:latest
platform: linux/amd64
hostname: spark-worker-1
container_name: spark-worker-1
@@ -199,7 +199,7 @@ services:
- "hive-metastore-postgresql"
- "namenode"
adhoc-1:
- image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkadhoc_3.5.3:latest
+ image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkadhoc_3.5.9:latest
platform: linux/amd64
hostname: adhoc-1
container_name: adhoc-1
@@ -221,7 +221,7 @@ services:
volumes:
- ${HUDI_WS}:/var/hoodie/ws
adhoc-2:
- image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkadhoc_3.5.3:latest
+ image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkadhoc_3.5.9:latest
platform: linux/amd64
hostname: adhoc-2
container_name: adhoc-2
diff --git a/docker/compose/docker-compose_hadoop334_hive313_spark353_arm64.yml
b/docker/compose/docker-compose_hadoop334_hive313_spark359_arm64.yml
similarity index 96%
rename from docker/compose/docker-compose_hadoop334_hive313_spark353_arm64.yml
rename to docker/compose/docker-compose_hadoop334_hive313_spark359_arm64.yml
index 4aecae8e30ae..d40f97ccc242 100644
--- a/docker/compose/docker-compose_hadoop334_hive313_spark353_arm64.yml
+++ b/docker/compose/docker-compose_hadoop334_hive313_spark359_arm64.yml
@@ -20,7 +20,7 @@ services:
hostname: namenode
container_name: namenode
environment:
- - CLUSTER_NAME=hudi_hadoop334_hive313_spark353
+ - CLUSTER_NAME=hudi_hadoop334_hive313_spark359
ports:
- "50070:50070"
- "8020:8020"
@@ -38,7 +38,7 @@ services:
container_name: datanode1
hostname: datanode1
environment:
- - CLUSTER_NAME=hudi_hadoop334_hive313_spark353
+ - CLUSTER_NAME=hudi_hadoop334_hive313_spark359
env_file:
- ./hadoop.env
ports:
@@ -61,7 +61,7 @@ services:
hostname: historyserver
container_name: historyserver
environment:
- - CLUSTER_NAME=hudi_hadoop334_hive313_spark353
+ - CLUSTER_NAME=hudi_hadoop334_hive313_spark359
depends_on:
- "namenode"
links:
@@ -148,7 +148,7 @@ services:
- ALLOW_PLAINTEXT_LISTENER=yes
sparkmaster:
- image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkmaster_3.5.3:latest
+ image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkmaster_3.5.9:latest
hostname: sparkmaster
container_name: sparkmaster
env_file:
@@ -169,7 +169,7 @@ services:
- "namenode"
spark-worker-1:
- image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkworker_3.5.3:latest
+ image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkworker_3.5.9:latest
hostname: spark-worker-1
container_name: spark-worker-1
env_file:
@@ -187,7 +187,7 @@ services:
- "namenode"
adhoc-1:
- image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkadhoc_3.5.3:latest
+ image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkadhoc_3.5.9:latest
hostname: adhoc-1
container_name: adhoc-1
env_file:
@@ -207,7 +207,7 @@ services:
- ${HUDI_WS}:/var/hoodie/ws
adhoc-2:
- image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkadhoc_3.5.3:latest
+ image: apachehudi/hudi-hadoop_3.3.4-hive_3.1.3-sparkadhoc_3.5.9:latest
hostname: adhoc-2
container_name: adhoc-2
env_file:
diff --git a/docker/hoodie/hadoop/pom.xml b/docker/hoodie/hadoop/pom.xml
index 25a934fafd3e..3d8f6b77ed73 100644
--- a/docker/hoodie/hadoop/pom.xml
+++ b/docker/hoodie/hadoop/pom.xml
@@ -67,7 +67,7 @@
<properties>
<skipITs>false</skipITs>
<docker.build.skip>true</docker.build.skip>
- <docker.spark.version>3.5.3</docker.spark.version>
+ <docker.spark.version>3.5.9</docker.spark.version>
<docker.hive.version>2.3.3</docker.hive.version>
<docker.hadoop.version>2.8.4</docker.hadoop.version>
<docker.presto.version>0.271</docker.presto.version>
diff --git a/docker/hoodie/hadoop/spark_base/Dockerfile
b/docker/hoodie/hadoop/spark_base/Dockerfile
index cfbf606db95c..c6c2885aaf1b 100644
--- a/docker/hoodie/hadoop/spark_base/Dockerfile
+++ b/docker/hoodie/hadoop/spark_base/Dockerfile
@@ -23,7 +23,7 @@ ENV ENABLE_INIT_DAEMON=true
ENV INIT_DAEMON_BASE_URI=http://identifier/init-daemon
ENV INIT_DAEMON_STEP=spark_master_init
-ARG SPARK_VERSION=3.5.3
+ARG SPARK_VERSION=3.5.9
ARG SPARK_HADOOP_VERSION=3
ENV SPARK_VERSION=${SPARK_VERSION}
diff --git a/docker/hoodie/hadoop/sparkadhoc/Dockerfile
b/docker/hoodie/hadoop/sparkadhoc/Dockerfile
index 7c8e900fca53..69b4d488db15 100644
--- a/docker/hoodie/hadoop/sparkadhoc/Dockerfile
+++ b/docker/hoodie/hadoop/sparkadhoc/Dockerfile
@@ -17,7 +17,7 @@
ARG HADOOP_VERSION=3.3.4
ARG HIVE_VERSION=3.1.3
-ARG SPARK_VERSION=3.5.3
+ARG SPARK_VERSION=3.5.9
FROM
apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION}
ARG PRESTO_VERSION=0.268
diff --git a/docker/hoodie/hadoop/sparkmaster/Dockerfile
b/docker/hoodie/hadoop/sparkmaster/Dockerfile
index 9acaf027606d..679530a26b3a 100644
--- a/docker/hoodie/hadoop/sparkmaster/Dockerfile
+++ b/docker/hoodie/hadoop/sparkmaster/Dockerfile
@@ -17,7 +17,7 @@
ARG HADOOP_VERSION=3.3.4
ARG HIVE_VERSION=3.1.3
-ARG SPARK_VERSION=3.5.3
+ARG SPARK_VERSION=3.5.9
FROM
apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION}
COPY master.sh /opt/spark
diff --git a/docker/hoodie/hadoop/sparkworker/Dockerfile
b/docker/hoodie/hadoop/sparkworker/Dockerfile
index 5b03834067ec..d8ec6fb9c362 100644
--- a/docker/hoodie/hadoop/sparkworker/Dockerfile
+++ b/docker/hoodie/hadoop/sparkworker/Dockerfile
@@ -17,7 +17,7 @@
ARG HADOOP_VERSION=3.3.4
ARG HIVE_VERSION=3.1.3
-ARG SPARK_VERSION=3.5.3
+ARG SPARK_VERSION=3.5.9
FROM
apachehudi/hudi-hadoop_${HADOOP_VERSION}-hive_${HIVE_VERSION}-sparkbase_${SPARK_VERSION}
COPY worker.sh /opt/spark
diff --git a/docker/setup_demo.sh b/docker/setup_demo.sh
index 3898852c9009..4346a6db42ca 100755
--- a/docker/setup_demo.sh
+++ b/docker/setup_demo.sh
@@ -19,9 +19,9 @@
SCRIPT_PATH=$(cd `dirname $0`; pwd)
HUDI_DEMO_ENV=$1
WS_ROOT=`dirname $SCRIPT_PATH`
-COMPOSE_FILE_NAME="docker-compose_hadoop334_hive313_spark353_amd64.yml"
+COMPOSE_FILE_NAME="docker-compose_hadoop334_hive313_spark359_amd64.yml"
if [ "$(uname -m)" = "arm64" ]; then
- COMPOSE_FILE_NAME="docker-compose_hadoop334_hive313_spark353_arm64.yml"
+ COMPOSE_FILE_NAME="docker-compose_hadoop334_hive313_spark359_arm64.yml"
fi
# restart cluster
HUDI_WS=${WS_ROOT} docker compose -f
${SCRIPT_PATH}/compose/${COMPOSE_FILE_NAME} down
diff --git a/docker/stop_demo.sh b/docker/stop_demo.sh
index bd9680f0fa81..c46eb5a2eb40 100755
--- a/docker/stop_demo.sh
+++ b/docker/stop_demo.sh
@@ -20,9 +20,9 @@ SCRIPT_PATH=$(cd `dirname $0`; pwd)
HUDI_DEMO_ENV=$1
# set up root directory
WS_ROOT=`dirname $SCRIPT_PATH`
-COMPOSE_FILE_NAME="docker-compose_hadoop334_hive313_spark353_amd64.yml"
+COMPOSE_FILE_NAME="docker-compose_hadoop334_hive313_spark359_amd64.yml"
if [ "$(uname -m)" = "arm64" ]; then
- COMPOSE_FILE_NAME="docker-compose_hadoop334_hive313_spark353_arm64.yml"
+ COMPOSE_FILE_NAME="docker-compose_hadoop334_hive313_spark359_arm64.yml"
fi
# shut down cluster
HUDI_WS=${WS_ROOT} docker compose -f
${SCRIPT_PATH}/compose/${COMPOSE_FILE_NAME} down
diff --git a/hudi-integ-test/pom.xml b/hudi-integ-test/pom.xml
index 0dc21a26f7f5..95ccd2a96f9f 100644
--- a/hudi-integ-test/pom.xml
+++ b/hudi-integ-test/pom.xml
@@ -410,7 +410,7 @@
<properties>
<dockerCompose.envFile>${project.basedir}/compose_env</dockerCompose.envFile>
<dockerCompose.file>
-
${project.basedir}/../docker/compose/docker-compose_hadoop284_hive2310_spark353_amd64.yml
+
${project.basedir}/../docker/compose/docker-compose_hadoop284_hive2310_spark359_amd64.yml
</dockerCompose.file>
<docker.compose.skip>${skipITs}</docker.compose.skip>
<main.basedir>${project.parent.basedir}</main.basedir>
@@ -570,7 +570,7 @@
<id>m1-mac</id>
<properties>
<dockerCompose.file>
-
${project.basedir}/../docker/compose/docker-compose_hadoop284_hive2310_spark353_arm64.yml
+
${project.basedir}/../docker/compose/docker-compose_hadoop284_hive2310_spark359_arm64.yml
</dockerCompose.file>
</properties>
<activation>
