This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 8240b6a2cd8 [HUDI-7113] Update release scripts and docs for Spark 3.5
support (#10123)
8240b6a2cd8 is described below
commit 8240b6a2cd80815f1c8a2f8af0894c2e28d3b715
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri Nov 17 11:20:57 2023 -0800
[HUDI-7113] Update release scripts and docs for Spark 3.5 support (#10123)
---
README.md | 9 +++++----
scripts/release/deploy_staging_jars.sh | 8 ++++++--
scripts/release/validate_staged_bundles.sh | 4 ++--
3 files changed, 13 insertions(+), 8 deletions(-)
diff --git a/README.md b/README.md
index 20016f689ad..6645f55b49b 100644
--- a/README.md
+++ b/README.md
@@ -66,8 +66,8 @@ git clone https://github.com/apache/hudi.git && cd hudi
mvn clean package -DskipTests
# Start command
-spark-3.2.3-bin-hadoop3.2/bin/spark-shell \
- --jars `ls
packaging/hudi-spark-bundle/target/hudi-spark3.2-bundle_2.12-*.*.*-SNAPSHOT.jar`
\
+spark-3.5.0-bin-hadoop3/bin/spark-shell \
+ --jars `ls
packaging/hudi-spark-bundle/target/hudi-spark3.5-bundle_2.12-*.*.*-SNAPSHOT.jar`
\
--conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer' \
--conf
'spark.sql.extensions=org.apache.spark.sql.hudi.HoodieSparkSessionExtension' \
--conf
'spark.sql.catalog.spark_catalog=org.apache.spark.sql.hudi.catalog.HoodieCatalog'
\
@@ -85,7 +85,7 @@ mvn clean javadoc:aggregate -Pjavadocs
### Build with different Spark versions
The default Spark 2.x version supported is 2.4.4. The default Spark 3.x
version, corresponding to `spark3` profile is
-3.4.0. The default Scala version is 2.12. Refer to the table below for
building with different Spark and Scala versions.
+3.5.0. The default Scala version is 2.12. Refer to the table below for
building with different Spark and Scala versions.
| Maven build options | Expected Spark bundle jar name |
Notes |
|:--------------------------|:---------------------------------------------|:-------------------------------------------------|
@@ -96,9 +96,10 @@ The default Spark 2.x version supported is 2.4.4. The
default Spark 3.x version,
| `-Dspark3.2` | hudi-spark3.2-bundle_2.12 |
For Spark 3.2.x and Scala 2.12 (same as default) |
| `-Dspark3.3` | hudi-spark3.3-bundle_2.12 |
For Spark 3.3.x and Scala 2.12 |
| `-Dspark3.4` | hudi-spark3.4-bundle_2.12 |
For Spark 3.4.x and Scala 2.12 |
+| `-Dspark3.5` | hudi-spark3.5-bundle_2.12 |
For Spark 3.5.x and Scala 2.12 |
| `-Dspark2 -Dscala-2.11` | hudi-spark-bundle_2.11 (legacy bundle name) |
For Spark 2.4.4 and Scala 2.11 |
| `-Dspark2 -Dscala-2.12` | hudi-spark-bundle_2.12 (legacy bundle name) |
For Spark 2.4.4 and Scala 2.12 |
-| `-Dspark3` | hudi-spark3-bundle_2.12 (legacy bundle name) |
For Spark 3.4.x and Scala 2.12 |
+| `-Dspark3` | hudi-spark3-bundle_2.12 (legacy bundle name) |
For Spark 3.5.x and Scala 2.12 |
For example,
```
diff --git a/scripts/release/deploy_staging_jars.sh
b/scripts/release/deploy_staging_jars.sh
index 4e54d30a2a9..22596809e37 100755
--- a/scripts/release/deploy_staging_jars.sh
+++ b/scripts/release/deploy_staging_jars.sh
@@ -66,9 +66,13 @@ declare -a ALL_VERSION_OPTS=(
"-Dscala-2.12 -Dspark3.3 -pl
hudi-spark-datasource/hudi-spark3.3.x,packaging/hudi-spark-bundle -am"
# For Spark 3.4, Scala 2.12:
# hudi-spark3.4.x_2.12
-# hudi-cli-bundle_2.12
# hudi-spark3.4-bundle_2.12
-"-Dscala-2.12 -Dspark3.4 -pl
hudi-spark-datasource/hudi-spark3.4.x,packaging/hudi-spark-bundle,packaging/hudi-cli-bundle
-am"
+"-Dscala-2.12 -Dspark3.4 -pl
hudi-spark-datasource/hudi-spark3.4.x,packaging/hudi-spark-bundle -am"
+# For Spark 3.5, Scala 2.12:
+# hudi-spark3.5.x_2.12
+# hudi-cli-bundle_2.12
+# hudi-spark3.5-bundle_2.12
+"-Dscala-2.12 -Dspark3.5 -pl
hudi-spark-datasource/hudi-spark3.5.x,packaging/hudi-spark-bundle,packaging/hudi-cli-bundle
-am"
# For Spark 3.1, Scala 2.12:
# All other modules and bundles using avro 1.8
"-Dscala-2.12 -Dspark3.1"
diff --git a/scripts/release/validate_staged_bundles.sh
b/scripts/release/validate_staged_bundles.sh
index 866b8cee335..579dc2410d3 100755
--- a/scripts/release/validate_staged_bundles.sh
+++ b/scripts/release/validate_staged_bundles.sh
@@ -36,8 +36,8 @@ declare -a bundles=("hudi-aws-bundle" "hudi-cli-bundle_2.11"
"hudi-cli-bundle_2.
"hudi-flink1.15-bundle" "hudi-flink1.16-bundle" "hudi-flink1.17-bundle"
"hudi-flink1.18-bundle" "hudi-gcp-bundle" "hudi-hadoop-mr-bundle"
"hudi-hive-sync-bundle" "hudi-integ-test-bundle"
"hudi-kafka-connect-bundle" "hudi-metaserver-server-bundle"
"hudi-presto-bundle" "hudi-spark-bundle_2.11" "hudi-spark-bundle_2.12"
"hudi-spark2.4-bundle_2.11" "hudi-spark2.4-bundle_2.12"
"hudi-spark3-bundle_2.12" "hudi-spark3.0-bundle_2.12"
"hudi-spark3.1-bundle_2.12"
-"hudi-spark3.2-bundle_2.12" "hudi-spark3.3-bundle_2.12"
"hudi-spark3.4-bundle_2.12" "hudi-timeline-server-bundle" "hudi-trino-bundle"
-"hudi-utilities-bundle_2.11" "hudi-utilities-bundle_2.12"
"hudi-utilities-slim-bundle_2.11"
+"hudi-spark3.2-bundle_2.12" "hudi-spark3.3-bundle_2.12"
"hudi-spark3.4-bundle_2.12" "hudi-spark3.5-bundle_2.12"
"hudi-timeline-server-bundle"
+"hudi-trino-bundle" "hudi-utilities-bundle_2.11" "hudi-utilities-bundle_2.12"
"hudi-utilities-slim-bundle_2.11"
"hudi-utilities-slim-bundle_2.12")
NOW=$(date +%s)
