This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new dccb129ea045 [SPARK-50493][SPARK-37687][BUILD] Migrate
kubernetes-client from `6.x` to `7.x`
dccb129ea045 is described below
commit dccb129ea045cfab36d8eb7a6525a6875c46e184
Author: panbingkun <[email protected]>
AuthorDate: Fri Jan 3 08:20:39 2025 -0800
[SPARK-50493][SPARK-37687][BUILD] Migrate kubernetes-client from `6.x` to
`7.x`
### What changes were proposed in this pull request?
The pr aims to migrate kubernetes-client from `6.x` to `7.x`, includes:
- upgrade `kubernetes-client` from `6.13.2` to `7.0.1`.
- switch default `HttpClient` implementation from `OkHttp` to `Vert.x`.
### Why are the changes needed?
- The full release notes:
https://github.com/fabric8io/kubernetes-client/releases/tag/v7.0.1
https://github.com/fabric8io/kubernetes-client/releases/tag/v7.0.0
- [Vert.x as default HttpClient
implementation](https://github.com/fabric8io/kubernetes-client/blob/v7.0.0/doc/MIGRATION-v7.md#vertx-as-default-httpclient-implementation-)
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass GA.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #49159 from panbingkun/k8s_client_7_vert_x.
Authored-by: panbingkun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
LICENSE-binary | 4 ++
dev/deps/spark-deps-hadoop-3-hive-2.3 | 61 ++++++++++++----------
hadoop-cloud/pom.xml | 12 +++++
pom.xml | 8 +--
resource-managers/kubernetes/core/pom.xml | 7 +--
.../deploy/k8s/SparkKubernetesClientFactory.scala | 18 +------
.../deploy/k8s/features/VolcanoFeatureStep.scala | 2 +-
.../k8s/features/VolcanoFeatureStepSuite.scala | 2 +-
.../k8s/integrationtest/VolcanoTestsSuite.scala | 2 +-
9 files changed, 55 insertions(+), 61 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index 40d28fbe71e6..5cf099cb4d3c 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -286,6 +286,10 @@ io.netty:netty-transport-classes-kqueue
io.netty:netty-transport-native-epoll
io.netty:netty-transport-native-kqueue
io.netty:netty-transport-native-unix-common
+io.vertx:vertx-auth-common
+io.vertx:vertx-core
+io.vertx:vertx-web-client
+io.vertx:vertx-web-common
jakarta.inject:jakarta.inject-api
jakarta.validation:jakarta.validation-api
javax.jdo:jdo-api
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 8ce2b4d36299..d6db12457eee 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -158,31 +158,31 @@ jsr305/3.0.0//jsr305-3.0.0.jar
jta/1.1//jta-1.1.jar
jul-to-slf4j/2.0.16//jul-to-slf4j-2.0.16.jar
kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client-api/6.13.4//kubernetes-client-api-6.13.4.jar
-kubernetes-client/6.13.4//kubernetes-client-6.13.4.jar
-kubernetes-httpclient-okhttp/6.13.4//kubernetes-httpclient-okhttp-6.13.4.jar
-kubernetes-model-admissionregistration/6.13.4//kubernetes-model-admissionregistration-6.13.4.jar
-kubernetes-model-apiextensions/6.13.4//kubernetes-model-apiextensions-6.13.4.jar
-kubernetes-model-apps/6.13.4//kubernetes-model-apps-6.13.4.jar
-kubernetes-model-autoscaling/6.13.4//kubernetes-model-autoscaling-6.13.4.jar
-kubernetes-model-batch/6.13.4//kubernetes-model-batch-6.13.4.jar
-kubernetes-model-certificates/6.13.4//kubernetes-model-certificates-6.13.4.jar
-kubernetes-model-common/6.13.4//kubernetes-model-common-6.13.4.jar
-kubernetes-model-coordination/6.13.4//kubernetes-model-coordination-6.13.4.jar
-kubernetes-model-core/6.13.4//kubernetes-model-core-6.13.4.jar
-kubernetes-model-discovery/6.13.4//kubernetes-model-discovery-6.13.4.jar
-kubernetes-model-events/6.13.4//kubernetes-model-events-6.13.4.jar
-kubernetes-model-extensions/6.13.4//kubernetes-model-extensions-6.13.4.jar
-kubernetes-model-flowcontrol/6.13.4//kubernetes-model-flowcontrol-6.13.4.jar
-kubernetes-model-gatewayapi/6.13.4//kubernetes-model-gatewayapi-6.13.4.jar
-kubernetes-model-metrics/6.13.4//kubernetes-model-metrics-6.13.4.jar
-kubernetes-model-networking/6.13.4//kubernetes-model-networking-6.13.4.jar
-kubernetes-model-node/6.13.4//kubernetes-model-node-6.13.4.jar
-kubernetes-model-policy/6.13.4//kubernetes-model-policy-6.13.4.jar
-kubernetes-model-rbac/6.13.4//kubernetes-model-rbac-6.13.4.jar
-kubernetes-model-resource/6.13.4//kubernetes-model-resource-6.13.4.jar
-kubernetes-model-scheduling/6.13.4//kubernetes-model-scheduling-6.13.4.jar
-kubernetes-model-storageclass/6.13.4//kubernetes-model-storageclass-6.13.4.jar
+kubernetes-client-api/7.0.1//kubernetes-client-api-7.0.1.jar
+kubernetes-client/7.0.1//kubernetes-client-7.0.1.jar
+kubernetes-httpclient-vertx/7.0.1//kubernetes-httpclient-vertx-7.0.1.jar
+kubernetes-model-admissionregistration/7.0.1//kubernetes-model-admissionregistration-7.0.1.jar
+kubernetes-model-apiextensions/7.0.1//kubernetes-model-apiextensions-7.0.1.jar
+kubernetes-model-apps/7.0.1//kubernetes-model-apps-7.0.1.jar
+kubernetes-model-autoscaling/7.0.1//kubernetes-model-autoscaling-7.0.1.jar
+kubernetes-model-batch/7.0.1//kubernetes-model-batch-7.0.1.jar
+kubernetes-model-certificates/7.0.1//kubernetes-model-certificates-7.0.1.jar
+kubernetes-model-common/7.0.1//kubernetes-model-common-7.0.1.jar
+kubernetes-model-coordination/7.0.1//kubernetes-model-coordination-7.0.1.jar
+kubernetes-model-core/7.0.1//kubernetes-model-core-7.0.1.jar
+kubernetes-model-discovery/7.0.1//kubernetes-model-discovery-7.0.1.jar
+kubernetes-model-events/7.0.1//kubernetes-model-events-7.0.1.jar
+kubernetes-model-extensions/7.0.1//kubernetes-model-extensions-7.0.1.jar
+kubernetes-model-flowcontrol/7.0.1//kubernetes-model-flowcontrol-7.0.1.jar
+kubernetes-model-gatewayapi/7.0.1//kubernetes-model-gatewayapi-7.0.1.jar
+kubernetes-model-metrics/7.0.1//kubernetes-model-metrics-7.0.1.jar
+kubernetes-model-networking/7.0.1//kubernetes-model-networking-7.0.1.jar
+kubernetes-model-node/7.0.1//kubernetes-model-node-7.0.1.jar
+kubernetes-model-policy/7.0.1//kubernetes-model-policy-7.0.1.jar
+kubernetes-model-rbac/7.0.1//kubernetes-model-rbac-7.0.1.jar
+kubernetes-model-resource/7.0.1//kubernetes-model-resource-7.0.1.jar
+kubernetes-model-scheduling/7.0.1//kubernetes-model-scheduling-7.0.1.jar
+kubernetes-model-storageclass/7.0.1//kubernetes-model-storageclass-7.0.1.jar
lapack/3.0.3//lapack-3.0.3.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
libfb303/0.9.3//libfb303-0.9.3.jar
@@ -193,7 +193,6 @@ log4j-api/2.24.3//log4j-api-2.24.3.jar
log4j-core/2.24.3//log4j-core-2.24.3.jar
log4j-layout-template-json/2.24.3//log4j-layout-template-json-2.24.3.jar
log4j-slf4j2-impl/2.24.3//log4j-slf4j2-impl-2.24.3.jar
-logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar
lz4-java/1.8.0//lz4-java-1.8.0.jar
metrics-core/4.2.29//metrics-core-4.2.29.jar
metrics-graphite/4.2.29//metrics-graphite-4.2.29.jar
@@ -203,6 +202,7 @@ metrics-jvm/4.2.29//metrics-jvm-4.2.29.jar
minlog/1.3.0//minlog-1.3.0.jar
netty-all/4.1.115.Final//netty-all-4.1.115.Final.jar
netty-buffer/4.1.115.Final//netty-buffer-4.1.115.Final.jar
+netty-codec-dns/4.1.115.Final//netty-codec-dns-4.1.115.Final.jar
netty-codec-http/4.1.115.Final//netty-codec-http-4.1.115.Final.jar
netty-codec-http2/4.1.115.Final//netty-codec-http2-4.1.115.Final.jar
netty-codec-socks/4.1.115.Final//netty-codec-socks-4.1.115.Final.jar
@@ -210,6 +210,7 @@ netty-codec/4.1.115.Final//netty-codec-4.1.115.Final.jar
netty-common/4.1.115.Final//netty-common-4.1.115.Final.jar
netty-handler-proxy/4.1.115.Final//netty-handler-proxy-4.1.115.Final.jar
netty-handler/4.1.115.Final//netty-handler-4.1.115.Final.jar
+netty-resolver-dns/4.1.115.Final//netty-resolver-dns-4.1.115.Final.jar
netty-resolver/4.1.115.Final//netty-resolver-4.1.115.Final.jar
netty-tcnative-boringssl-static/2.0.69.Final/linux-aarch_64/netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar
netty-tcnative-boringssl-static/2.0.69.Final/linux-x86_64/netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar
@@ -258,7 +259,7 @@
scala-parser-combinators_2.13/2.4.0//scala-parser-combinators_2.13-2.4.0.jar
scala-reflect/2.13.15//scala-reflect-2.13.15.jar
scala-xml_2.13/2.3.0//scala-xml_2.13-2.3.0.jar
slf4j-api/2.0.16//slf4j-api-2.0.16.jar
-snakeyaml-engine/2.7//snakeyaml-engine-2.7.jar
+snakeyaml-engine/2.8//snakeyaml-engine-2.8.jar
snakeyaml/2.3//snakeyaml-2.3.jar
snappy-java/1.1.10.7//snappy-java-1.1.10.7.jar
spire-macros_2.13/0.18.0//spire-macros_2.13-0.18.0.jar
@@ -272,11 +273,15 @@ threeten-extra/1.7.1//threeten-extra-1.7.1.jar
tink/1.16.0//tink-1.16.0.jar
transaction-api/1.1//transaction-api-1.1.jar
univocity-parsers/2.9.1//univocity-parsers-2.9.1.jar
+vertx-auth-common/4.5.11//vertx-auth-common-4.5.11.jar
+vertx-core/4.5.11//vertx-core-4.5.11.jar
+vertx-web-client/4.5.11//vertx-web-client-4.5.11.jar
+vertx-web-common/4.5.11//vertx-web-common-4.5.11.jar
wildfly-openssl/1.1.3.Final//wildfly-openssl-1.1.3.Final.jar
xbean-asm9-shaded/4.26//xbean-asm9-shaded-4.26.jar
xmlschema-core/2.3.1//xmlschema-core-2.3.1.jar
xz/1.10//xz-1.10.jar
-zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar
+zjsonpatch/7.0.1//zjsonpatch-7.0.1.jar
zookeeper-jute/3.9.3//zookeeper-jute-3.9.3.jar
zookeeper/3.9.3//zookeeper-3.9.3.jar
zstd-jni/1.5.6-8//zstd-jni-1.5.6-8.jar
diff --git a/hadoop-cloud/pom.xml b/hadoop-cloud/pom.xml
index 9c2f21e7ab61..336108154924 100644
--- a/hadoop-cloud/pom.xml
+++ b/hadoop-cloud/pom.xml
@@ -34,6 +34,8 @@
</description>
<properties>
<sbt.project.name>hadoop-cloud</sbt.project.name>
+ <okhttp.version>3.12.12</okhttp.version>
+ <okio.version>1.17.6</okio.version>
</properties>
<dependencies>
@@ -191,6 +193,16 @@
<version>${jetty.version}</version>
<scope>${hadoop.deps.scope}</scope>
</dependency>
+ <dependency>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>okhttp</artifactId>
+ <version>${okhttp.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.squareup.okio</groupId>
+ <artifactId>okio</artifactId>
+ <version>${okio.version}</version>
+ </dependency>
</dependencies>
<build>
diff --git a/pom.xml b/pom.xml
index 05ebba16270d..ba8f42a3b181 100644
--- a/pom.xml
+++ b/pom.xml
@@ -235,8 +235,7 @@
<!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
<leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
- <kubernetes-client.version>6.13.4</kubernetes-client.version>
- <okio.version>1.17.6</okio.version>
+ <kubernetes-client.version>7.0.1</kubernetes-client.version>
<test.java.home>${java.home}</test.java.home>
@@ -2594,11 +2593,6 @@
<artifactId>javax.servlet-api</artifactId>
<version>${javaxservlet.version}</version>
</dependency>
- <dependency>
- <groupId>com.squareup.okio</groupId>
- <artifactId>okio</artifactId>
- <version>${okio.version}</version>
- </dependency>
</dependencies>
</dependencyManagement>
diff --git a/resource-managers/kubernetes/core/pom.xml
b/resource-managers/kubernetes/core/pom.xml
index 211c6c93b967..814a3e1c595b 100644
--- a/resource-managers/kubernetes/core/pom.xml
+++ b/resource-managers/kubernetes/core/pom.xml
@@ -37,7 +37,7 @@
<dependencies>
<dependency>
<groupId>io.fabric8</groupId>
- <artifactId>volcano-model-v1beta1</artifactId>
+ <artifactId>volcano-model</artifactId>
<version>${kubernetes-client.version}</version>
</dependency>
<dependency>
@@ -105,11 +105,6 @@
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>io.fabric8</groupId>
- <artifactId>kubernetes-httpclient-okhttp</artifactId>
- <version>${kubernetes-client.version}</version>
- </dependency>
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>kubernetes-client</artifactId>
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
index 2c28dc380046..557bf01cbdba 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
@@ -24,10 +24,7 @@ import com.google.common.io.Files
import io.fabric8.kubernetes.client.{ConfigBuilder, KubernetesClient,
KubernetesClientBuilder}
import
io.fabric8.kubernetes.client.Config.KUBERNETES_REQUEST_RETRY_BACKOFFLIMIT_SYSTEM_PROPERTY
import io.fabric8.kubernetes.client.Config.autoConfigure
-import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory
import io.fabric8.kubernetes.client.utils.Utils.getSystemPropertyOrEnvVar
-import okhttp3.Dispatcher
-import okhttp3.OkHttpClient
import org.apache.spark.SparkConf
import org.apache.spark.annotation.{DeveloperApi, Since, Stable}
@@ -35,7 +32,6 @@ import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.internal.{Logging, MDC}
import org.apache.spark.internal.LogKeys.K8S_CONTEXT
import org.apache.spark.internal.config.ConfigEntry
-import org.apache.spark.util.ThreadUtils
/**
* :: DeveloperApi ::
@@ -78,10 +74,6 @@ object SparkKubernetesClientFactory extends Logging {
.getOption(s"$kubernetesAuthConfPrefix.$CLIENT_KEY_FILE_CONF_SUFFIX")
val clientCertFile = sparkConf
.getOption(s"$kubernetesAuthConfPrefix.$CLIENT_CERT_FILE_CONF_SUFFIX")
- // TODO(SPARK-37687): clean up direct usage of OkHttpClient, see also:
- // https://github.com/fabric8io/kubernetes-client/issues/3547
- val dispatcher = new Dispatcher(
- ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher"))
// Allow for specifying a context used to auto-configure from the users
K8S config file
val kubeContext = sparkConf.get(KUBERNETES_CONTEXT).filter(_.nonEmpty)
@@ -117,17 +109,9 @@ object SparkKubernetesClientFactory extends Logging {
}.withOption(namespace) {
(ns, configBuilder) => configBuilder.withNamespace(ns)
}.build()
- val factoryWithCustomDispatcher = new OkHttpClientFactory() {
- override protected def additionalConfig(builder: OkHttpClient.Builder):
Unit = {
- builder.dispatcher(dispatcher)
- }
- }
logDebug("Kubernetes client config: " +
new
ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(config))
- new KubernetesClientBuilder()
- .withHttpClientFactory(factoryWithCustomDispatcher)
- .withConfig(config)
- .build()
+ new KubernetesClientBuilder().withConfig(config).build()
}
private implicit class OptionConfigurableConfigBuilder(val configBuilder:
ConfigBuilder)
diff --git
a/resource-managers/kubernetes/core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
b/resource-managers/kubernetes/core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
index 314550713ef1..046d268df4e2 100644
---
a/resource-managers/kubernetes/core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
+++
b/resource-managers/kubernetes/core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
@@ -17,8 +17,8 @@
package org.apache.spark.deploy.k8s.features
import io.fabric8.kubernetes.api.model._
+import io.fabric8.volcano.api.model.scheduling.v1beta1.{PodGroup, PodGroupSpec}
import io.fabric8.volcano.client.DefaultVolcanoClient
-import io.fabric8.volcano.scheduling.v1beta1.{PodGroup, PodGroupSpec}
import org.apache.spark.deploy.k8s.{KubernetesConf, KubernetesDriverConf,
KubernetesExecutorConf, SparkPod}
import org.apache.spark.internal.Logging
diff --git
a/resource-managers/kubernetes/core/volcano/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
b/resource-managers/kubernetes/core/volcano/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
index dab414e0e19e..c92164993ef8 100644
---
a/resource-managers/kubernetes/core/volcano/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
+++
b/resource-managers/kubernetes/core/volcano/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
@@ -18,7 +18,7 @@ package org.apache.spark.deploy.k8s.features
import java.io.File
-import io.fabric8.volcano.scheduling.v1beta1.PodGroup
+import io.fabric8.volcano.api.model.scheduling.v1beta1.PodGroup
import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.deploy.k8s._
diff --git
a/resource-managers/kubernetes/integration-tests/volcano/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
b/resource-managers/kubernetes/integration-tests/volcano/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
index 2f414b72ee19..4a9dc135ecfd 100644
---
a/resource-managers/kubernetes/integration-tests/volcano/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
+++
b/resource-managers/kubernetes/integration-tests/volcano/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
@@ -28,8 +28,8 @@ import scala.concurrent.Future
import scala.jdk.CollectionConverters._
import io.fabric8.kubernetes.api.model.{HasMetadata, Pod, Quantity}
+import io.fabric8.volcano.api.model.scheduling.v1beta1.{Queue, QueueBuilder}
import io.fabric8.volcano.client.VolcanoClient
-import io.fabric8.volcano.scheduling.v1beta1.{Queue, QueueBuilder}
import org.scalatest.BeforeAndAfterEach
import org.scalatest.concurrent.Eventually
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
