This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git
The following commit(s) were added to refs/heads/main by this push:
new b8e4f36 [SPARK-58140] Upgrade Spark to 4.2.0
b8e4f36 is described below
commit b8e4f36c3084ade19c5dd37a98fe7b5eee6c1b2a
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Jul 15 07:32:58 2026 -0700
[SPARK-58140] Upgrade Spark to 4.2.0
### What changes were proposed in this pull request?
This PR aims to upgrade Apache Spark to 4.2.0.
### Why are the changes needed?
Apache Spark 4.2.0 is released officially.
- https://github.com/apache/spark/releases/tag/v4.2.0
- https://spark.apache.org/docs/4.2.0/
- https://repo1.maven.org/maven2/org/apache/spark/spark-core_2.13/4.2.0/
This PR also excludes the `leveldbjni` variants, too.
### Does this PR introduce _any_ user-facing change?
No behavior change.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #754 from dongjoon-hyun/SPARK-58140.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
gradle/libs.versions.toml | 2 +-
gradle/verification-metadata.xml | 116 +++++++++++++++++------------------
spark-operator/build.gradle | 2 +
spark-submission-worker/build.gradle | 1 +
4 files changed, 62 insertions(+), 59 deletions(-)
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 20727cf..56ce088 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -20,7 +20,7 @@ lombok = "1.18.46"
netty = "4.2.16.Final"
operator-sdk = "5.4.0"
dropwizard-metrics = "4.2.39"
-spark = "4.2.0-preview5"
+spark = "4.2.0"
hadoop = "3.5.0"
log4j = "2.26.0"
slf4j = "2.0.18"
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
index 43e38fa..f5ac0aa 100644
--- a/gradle/verification-metadata.xml
+++ b/gradle/verification-metadata.xml
@@ -2667,97 +2667,97 @@
<sha512
value="99ce348021aebc9f51e8dbf08318f27188a0da56c874f3d07b2ba416f227e7609b3a0e594447c0982bc23e1cc8850d66c1c9a243038974ca1b85051947ffcc66"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-common-utils-java_2.13"
version="4.2.0-preview5">
- <artifact name="spark-common-utils-java_2.13-4.2.0-preview5.jar">
- <sha512
value="301efaaa876099d5b90d8de35f30ba869f2dcb14c4b0c4493ad9fadeb1307eefdd7f0a0139022f563ef495eafd5f5cfa3970c409d69b153f6a8691cb56dbe776"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-common-utils-java_2.13"
version="4.2.0">
+ <artifact name="spark-common-utils-java_2.13-4.2.0.jar">
+ <sha512
value="39c944490b1e208ff48b21de6d2a948d7c3bf500c776ee1fb5b18e2921b62b699aaa3decaed8a92c12b9adaf9a1496973dc662f6d5a7ab010ae869a4b254eeb6"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-common-utils-java_2.13-4.2.0-preview5.pom">
- <sha512
value="a0bd6da3ed0d7ceeb45cbce6e4a34f53d0ddcf33fb67e368cb89b499a8275f7e0179fa38b9a49cfb7bfdc7e512ccce1b21c027c2f95cc04ca4690e25c254f2cf"
origin="Generated by Gradle"/>
+ <artifact name="spark-common-utils-java_2.13-4.2.0.pom">
+ <sha512
value="be4ef5c70f50d9fff3fcc0cd39c993a4b53ce4653fa7d8924d97dbff6c59080d3fd143c60f6328a741256548da0b78a84601117d4f9a4aff0b4c882cb497aa16"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-common-utils_2.13"
version="4.2.0-preview5">
- <artifact name="spark-common-utils_2.13-4.2.0-preview5.jar">
- <sha512
value="62d3228f4985218f07e841eb4b37c420e99c64f95dfdfc9f2b468b9d4c87a82e4f6603cd4bf27bef222d62f433ad2b70018a37d6eb88b8d541cb5e11b122306e"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-common-utils_2.13"
version="4.2.0">
+ <artifact name="spark-common-utils_2.13-4.2.0.jar">
+ <sha512
value="86a3a25d8e65c5bc3e5c073d7f4d9ea9143543e48b2c62dfc4a5faecb278fc0c704475280f3e55e9c32b6a743e142939f308cf371dd65625769803eb1f48cb0c"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-common-utils_2.13-4.2.0-preview5.pom">
- <sha512
value="1ecb4c89b68472f985d31d39e660519d9bd9223c8ee366fc159508eaf098988b0e7a84ee3bac32bef23725887126cd63e4a3eb35eb069bdf57c527291a46da87"
origin="Generated by Gradle"/>
+ <artifact name="spark-common-utils_2.13-4.2.0.pom">
+ <sha512
value="f41aeb5c59e6f2a21025c9a31c99eafc859adc410664cf058c27f19d5bdfd7d6cd827b32cb8615331bd0e73aca2d119e63a159c1846f2fee8eccd416f18b3b37"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-core_2.13"
version="4.2.0-preview5">
- <artifact name="spark-core_2.13-4.2.0-preview5.jar">
- <sha512
value="62240d5bd42c957738f6051d14cba597cab12785b969c51f07ba4d71f9d5e5e95fd9bc3415407ba6e8f64843fcd0d98a2fe6c74f05dfd85692a060ec93076445"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-core_2.13"
version="4.2.0">
+ <artifact name="spark-core_2.13-4.2.0.jar">
+ <sha512
value="cda41489aa5e807ecf7b5665f7f845847eb445869780b05c934577278890e4dd5899f10e7482c906a324191a013bc829cdd762c5f28614b94dfd0bbb8aa8963a"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-core_2.13-4.2.0-preview5.pom">
- <sha512
value="d230375b2d5140bf296bcdcca2d965e890ad0707485c790349e828264032c4ca2be949541c1f5774c5d6bd3abe47be6af4ffd51ff7a01153b33c3b32d67f9232"
origin="Generated by Gradle"/>
+ <artifact name="spark-core_2.13-4.2.0.pom">
+ <sha512
value="8fc8f2e1163521d65ce34d5cdd62d07164af5d95ad95dbd3676c6dcbf64b2585c08cab93b374f46119a2ed6e728105c6dfb038afc3e87553ccb424de35680707"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-kubernetes_2.13"
version="4.2.0-preview5">
- <artifact name="spark-kubernetes_2.13-4.2.0-preview5.jar">
- <sha512
value="fecfd95f10367ad588eec82dfd16f51a217d19bed1325f4432d4efe7c62a56622e73dc8f4991dc93647487e1a2e6c1ed6eb0851c40913101f52ced01dc7f71a5"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-kubernetes_2.13"
version="4.2.0">
+ <artifact name="spark-kubernetes_2.13-4.2.0.jar">
+ <sha512
value="4307cc68c9f794ce6ed909bbee21d10e0e154b1897dc3463c651bb6ed768396e047e45b0c7889d7fb64db2a17eb6cd68f53b633cad9395eb71aace68ba3bfc0e"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-kubernetes_2.13-4.2.0-preview5.pom">
- <sha512
value="5e278aa4dfd0c8d55b2836d09589aed540728f583e9487b8a50077621ed10e9626722da5c71871d7adef8f0c09c137d66fe7c3a7f200ecf204e8991ffb88bf11"
origin="Generated by Gradle"/>
+ <artifact name="spark-kubernetes_2.13-4.2.0.pom">
+ <sha512
value="e0766d295db328e97551376af01be7db5738c30d6d5bf89bb3f58057f3cefc30dfebea4f6a53600834733d0957ade1ed3f73dcd5bdfe70e3454f215159a9ee19"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-kvstore_2.13"
version="4.2.0-preview5">
- <artifact name="spark-kvstore_2.13-4.2.0-preview5.jar">
- <sha512
value="43554d7615471aaacd7d31160b24ef35324cb8fc44951368e9626701ee1388772c272587f3bd70fd57cd29a8786c06257ef6278e6281e7d1aaaa097a0b96b96f"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-kvstore_2.13"
version="4.2.0">
+ <artifact name="spark-kvstore_2.13-4.2.0.jar">
+ <sha512
value="222af7fd9b4b25112123eeb0afc05a620500b93e561dc4e0fcb4c9f220f3c5242d2c92f507f84f2e7c056662adc52ece3f74f9b1c395d7641cb49d021df3c7e6"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-kvstore_2.13-4.2.0-preview5.pom">
- <sha512
value="095d05b15417241b3431561fc7aaa0913259c5ee8e3f93a6f611271978b9e45e6f97103f137230d56e126a71d572aec7aaaeaf59a2b3541d2c055c6994d9fe3c"
origin="Generated by Gradle"/>
+ <artifact name="spark-kvstore_2.13-4.2.0.pom">
+ <sha512
value="97d47caaa846b5436494ff295370f215df36a707386c5da5eadfe808987d5457ac42f073760f8fdd8bb448c570b39ba152558c56f89daa9bb8350c81b89994e9"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-launcher_2.13"
version="4.2.0-preview5">
- <artifact name="spark-launcher_2.13-4.2.0-preview5.jar">
- <sha512
value="f773107cdbc80b02a1d011d7d2fa7b8a99447b5c2c446608406d3535fc09721a67b0be2558c7dc2157201a6c44cf77678fc6e29070b52b738c348a163f9c35fa"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-launcher_2.13"
version="4.2.0">
+ <artifact name="spark-launcher_2.13-4.2.0.jar">
+ <sha512
value="df113b4bd4eb814d0624cffb2b865424e1f9de9447f4d1494002c07f52bf2748d0dd065b49a7ac36fda58a0304538e5627f9eccd9b156d8f988573cb7edcadb7"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-launcher_2.13-4.2.0-preview5.pom">
- <sha512
value="386e25775e9ab9fc749df48dd150dfe03e3953d8d7806c4ad128751cd4248f40d7194b82bd90c0fb818396b71f4a0f4a3a9683454b21844e6c888c207741a9c8"
origin="Generated by Gradle"/>
+ <artifact name="spark-launcher_2.13-4.2.0.pom">
+ <sha512
value="0c492201499cd3c6a6c2679f4439106c753a0825df9908e99bf75344f7c9cd16be94476b14f0677518545c7245df592516426f73dc8d4b2fc656ade38bae98dc"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-network-common_2.13"
version="4.2.0-preview5">
- <artifact name="spark-network-common_2.13-4.2.0-preview5.jar">
- <sha512
value="a92e0f0a03566cb3df3fef85b8770338ff1bc178f1cd4013aa991f5e6af92dd814859eb521ceb9054b0c536312409e6eaa2a17e17e27302845ac8e8a71e44313"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-network-common_2.13"
version="4.2.0">
+ <artifact name="spark-network-common_2.13-4.2.0.jar">
+ <sha512
value="008fb02c27f3035a918acd9e26d71bc6932200fd74e01524aa3830a2ff2f5c7f7ff6218dd9a45d16ec27ee844e8c8e3d4f24506b6dff2d2427337e04886915f8"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-network-common_2.13-4.2.0-preview5.pom">
- <sha512
value="0e1d411fe63876865e131628763a8b6f69333198d8953a995d2ddb08ef357fa65c98a4aba2c6d8d561421ca91f5965ed42eceb9a795703b1a4cf60ac845b7846"
origin="Generated by Gradle"/>
+ <artifact name="spark-network-common_2.13-4.2.0.pom">
+ <sha512
value="df6c1e1058e90266faffd20d55aabf2bd6dc3b12bcc9a0556d43dda6c4f5f11baef088e6dc40c220f010bc1264e0d80170e3fc0ab7de9c224796f65ae9dfa8aa"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-network-shuffle_2.13"
version="4.2.0-preview5">
- <artifact name="spark-network-shuffle_2.13-4.2.0-preview5.jar">
- <sha512
value="4d322a1992fb716b05a36de3f820a4259d79442f564840af01433ea2be89813a6eee437817e969a6723cda9f932b22b508057c56f1723b1ffea7813ce6bc2d8c"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-network-shuffle_2.13"
version="4.2.0">
+ <artifact name="spark-network-shuffle_2.13-4.2.0.jar">
+ <sha512
value="b9a5764cfd0fb843aece837e41bb0e4b1792344e205547fa4ecd46d1576072b91e4a278cae12a8f470c5e9fcd0968577184ac13e0dbcb48648166cdf7bdbe08e"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-network-shuffle_2.13-4.2.0-preview5.pom">
- <sha512
value="a35c5f7119b876e0d8b0d541a96d8884d67bb420ea55ee6967f010bae996a0d86d12f3a76473b833e05dab2f516afdebb517ec4644472350e6cb190b7e38b2aa"
origin="Generated by Gradle"/>
+ <artifact name="spark-network-shuffle_2.13-4.2.0.pom">
+ <sha512
value="9bb81096b5106a782ba5e332e5735897f789acbaca6efc0d5bdde233610f531e8419c29382ffa0f84065b44de90d2cb5d46af1f9d5da9a8839cee316f5a9c5b1"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-parent_2.13"
version="4.2.0-preview5">
- <artifact name="spark-parent_2.13-4.2.0-preview5.pom">
- <sha512
value="dda5861d6be9c520e2769eaf80a98b8c82dc696f1847425354b655bc63c7d0ed2c516af3e29fd78f83a22d92cf89121223e5e39f4e699ae46f6fa86e22d025ab"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-parent_2.13"
version="4.2.0">
+ <artifact name="spark-parent_2.13-4.2.0.pom">
+ <sha512
value="c8862a2a4fef67e37e17959025c1caf2c7a66375fa4cfcb85ea964f6371f0d3b3c756cda0846514bc1d49d5454ced392ea7957f16758489db94fba945dc64958"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-tags_2.13"
version="4.2.0-preview5">
- <artifact name="spark-tags_2.13-4.2.0-preview5.jar">
- <sha512
value="405b92d12a5daf4120a6b76fc9f413ec265e63db5f87a69420e6591754af1c51c50112fe29fe5f102d546d655130d91480fad4e64f9215d852133798c3309544"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-tags_2.13"
version="4.2.0">
+ <artifact name="spark-tags_2.13-4.2.0.jar">
+ <sha512
value="0d67873a1d85140889475f1c6ec6105154a687cd50111128a709d853bbdca17da351a348c51a78d80365b0c94a4ad98c353d4d063cfcbc9dc70e73acfa9dd1cf"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-tags_2.13-4.2.0-preview5.pom">
- <sha512
value="f862c8ea256ecad3ffe04d0a1ad27bb023d5ae81b0e4930311a0bf5decd59b231cca69fe1a683adea55738cbce0b736ee4841d51a0c948b04ea3213885f7a9c7"
origin="Generated by Gradle"/>
+ <artifact name="spark-tags_2.13-4.2.0.pom">
+ <sha512
value="2e92e58892df9c6032d318d9f65bb04984b66cde4b4b80d741d71da6c7e48d4090df2255e0541ecccffec4719e16139837a68db2d8fed5fe8f37223ead2034e6"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-unsafe_2.13"
version="4.2.0-preview5">
- <artifact name="spark-unsafe_2.13-4.2.0-preview5.jar">
- <sha512
value="16a34d50c5586306ee1af5515ccf8d7b0b07786d3700e59291fc9615fdfa34113832323c91580e81b790b59c86e9d5effdc86c1775636efd30763805a958459d"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-unsafe_2.13"
version="4.2.0">
+ <artifact name="spark-unsafe_2.13-4.2.0.jar">
+ <sha512
value="bff98867e89fc016a7c45b7380248bb00709d3efa8e8c2414c0d44ae6b3eb037403965f9dedd10c36eaaa9a47b39ea9c90249728e471aca7d0d16a04a71f2dfe"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-unsafe_2.13-4.2.0-preview5.pom">
- <sha512
value="b8d8a579b24743c74637e4a2a60f29b130902dc1474e98e7cf405144b9d1d81c978881eadcb5a94077fde69ff1116dcf690c3b7c7fdb2c2a39077f3b517f38cc"
origin="Generated by Gradle"/>
+ <artifact name="spark-unsafe_2.13-4.2.0.pom">
+ <sha512
value="c62aaa7ee63d8c2145198851f39fa1f7c1b39c0e2b599076eb9e440cb2eb8ca4afa031556464d4e0af9f2314fb620234b4d6b99e8e1decb3248548c2e7555cce"
origin="Generated by Gradle"/>
</artifact>
</component>
- <component group="org.apache.spark" name="spark-variant_2.13"
version="4.2.0-preview5">
- <artifact name="spark-variant_2.13-4.2.0-preview5.jar">
- <sha512
value="512518be0704ecc285c1522c29d80bc0b84ad32235185107449062b76146bb07bc543a8ebf5532ebc76dacfa1e2f8a2f8e300c56cde8aadf5550222f8dbd624d"
origin="Generated by Gradle"/>
+ <component group="org.apache.spark" name="spark-variant_2.13"
version="4.2.0">
+ <artifact name="spark-variant_2.13-4.2.0.jar">
+ <sha512
value="ace55e359a9fb11ead19b0ef89bfb5c7c6c0ec9a5b99ef9f296fa4a33ea36a7b19b55183edbf252389a32ba9f36855c08191b096b0340835d7f705a028f9af2a"
origin="Generated by Gradle"/>
</artifact>
- <artifact name="spark-variant_2.13-4.2.0-preview5.pom">
- <sha512
value="738481fb537f8161ae39a64197a6591dac08facaf3d7d0688c6486c530e427cec87d49943923119b4a8e535c1a23b27fb21d0062acc3b2919781623e5e088cdd"
origin="Generated by Gradle"/>
+ <artifact name="spark-variant_2.13-4.2.0.pom">
+ <sha512
value="549e16daf8b4affe21539c519b7311aae0af852a26ed26c43524af27b2608fca746b2005258f70699b1d2307a305633842e52de6091bceced06924d66c47bc7c"
origin="Generated by Gradle"/>
</artifact>
</component>
<component group="org.apache.xbean" name="xbean" version="3.7">
diff --git a/spark-operator/build.gradle b/spark-operator/build.gradle
index 0f42a59..5c954ff 100644
--- a/spark-operator/build.gradle
+++ b/spark-operator/build.gradle
@@ -51,6 +51,7 @@ dependencies {
exclude group: "org.checkerframework"
exclude group: "org.fusesource.leveldbjni"
exclude group: "org.lz4"
+ exclude group: "org.openlabtesting.leveldbjni"
exclude group: "org.roaringbitmap"
exclude group: "org.rocksdb"
exclude group: "org.slf4j"
@@ -67,6 +68,7 @@ dependencies {
}
testImplementation(libs.spark.core) {
exclude group: "org.apache.logging.log4j"
+ exclude group: "org.openlabtesting.leveldbjni"
exclude group: "org.slf4j"
}
testImplementation platform(libs.junit.bom)
diff --git a/spark-submission-worker/build.gradle
b/spark-submission-worker/build.gradle
index 1fb68cc..b338fe4 100644
--- a/spark-submission-worker/build.gradle
+++ b/spark-submission-worker/build.gradle
@@ -40,6 +40,7 @@ dependencies {
exclude group: "org.checkerframework"
exclude group: "org.fusesource.leveldbjni"
exclude group: "org.lz4"
+ exclude group: "org.openlabtesting.leveldbjni"
exclude group: "org.roaringbitmap"
exclude group: "org.rocksdb"
exclude group: "org.slf4j"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
