This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 18f9e7e [SPARK-37258][K8S][BUILD] Upgrade kubernetes-client to 5.12.0
18f9e7e is described below
commit 18f9e7efac5100744f255b6c8ae267579cd8d9ce
Author: Yikun Jiang <[email protected]>
AuthorDate: Mon Jan 24 23:49:10 2022 -0800
[SPARK-37258][K8S][BUILD] Upgrade kubernetes-client to 5.12.0
### What changes were proposed in this pull request?
This patch aims to upgrade `kubernetes-client` from 5.10.2 to 5.12.0
Hightlight changes:
- v5.12.0: Update Fabric8 Kubernetes Model to v1.23.0, bump
jackson-datatype-jsr310 to
[2.13.1](https://github.com/fabric8io/kubernetes-client/commit/43562675d36a7240825453370f5d540370e84cbb).
- v5.11.0: Introduce the Volcano extension, it would be useful for users
who want to use Volcano as customized scheduler in Spark on K8S.
- v5.11.0: Breaking change: an abstraction layer added over okHttp
There are also some changes to make sure it is compatiable to create http
client with custom dispatcher according to
https://github.com/fabric8io/kubernetes-client/issues/3663#issuecomment-997214403
suggestion.
### Why are the changes needed?
This will bring several bug fixes and improvements (such as volcano
support, Fabric8 Kuberentes model 1.23 support), see more in:
- https://github.com/fabric8io/kubernetes-client/releases/tag/v5.11.0
- https://github.com/fabric8io/kubernetes-client/releases/tag/v5.11.1
- https://github.com/fabric8io/kubernetes-client/releases/tag/v5.11.2
- https://github.com/fabric8io/kubernetes-client/releases/tag/v5.12.0
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- Pass the CIs.
- Running a spark-pi job to validate manually.
Closes #34939 from Yikun/SPARK-37258.
Authored-by: Yikun Jiang <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
dev/deps/spark-deps-hadoop-2-hive-2.3 | 44 +++++++++++-----------
dev/deps/spark-deps-hadoop-3-hive-2.3 | 44 +++++++++++-----------
pom.xml | 2 +-
.../deploy/k8s/SparkKubernetesClientFactory.scala | 16 +++++---
.../spark/deploy/k8s/integrationtest/Utils.scala | 4 +-
5 files changed, 57 insertions(+), 53 deletions(-)
diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3
b/dev/deps/spark-deps-hadoop-2-hive-2.3
index 164eab0..5efdca9 100644
--- a/dev/deps/spark-deps-hadoop-2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2-hive-2.3
@@ -118,7 +118,7 @@ jackson-core/2.13.1//jackson-core-2.13.1.jar
jackson-databind/2.13.1//jackson-databind-2.13.1.jar
jackson-dataformat-cbor/2.13.1//jackson-dataformat-cbor-2.13.1.jar
jackson-dataformat-yaml/2.13.1//jackson-dataformat-yaml-2.13.1.jar
-jackson-datatype-jsr310/2.13.0//jackson-datatype-jsr310-2.13.0.jar
+jackson-datatype-jsr310/2.13.1//jackson-datatype-jsr310-2.13.1.jar
jackson-jaxrs/1.9.13//jackson-jaxrs-1.9.13.jar
jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-module-scala_2.12/2.13.1//jackson-module-scala_2.12-2.13.1.jar
@@ -162,27 +162,27 @@ jsr305/3.0.0//jsr305-3.0.0.jar
jta/1.1//jta-1.1.jar
jul-to-slf4j/1.7.32//jul-to-slf4j-1.7.32.jar
kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/5.10.2//kubernetes-client-5.10.2.jar
-kubernetes-model-admissionregistration/5.10.2//kubernetes-model-admissionregistration-5.10.2.jar
-kubernetes-model-apiextensions/5.10.2//kubernetes-model-apiextensions-5.10.2.jar
-kubernetes-model-apps/5.10.2//kubernetes-model-apps-5.10.2.jar
-kubernetes-model-autoscaling/5.10.2//kubernetes-model-autoscaling-5.10.2.jar
-kubernetes-model-batch/5.10.2//kubernetes-model-batch-5.10.2.jar
-kubernetes-model-certificates/5.10.2//kubernetes-model-certificates-5.10.2.jar
-kubernetes-model-common/5.10.2//kubernetes-model-common-5.10.2.jar
-kubernetes-model-coordination/5.10.2//kubernetes-model-coordination-5.10.2.jar
-kubernetes-model-core/5.10.2//kubernetes-model-core-5.10.2.jar
-kubernetes-model-discovery/5.10.2//kubernetes-model-discovery-5.10.2.jar
-kubernetes-model-events/5.10.2//kubernetes-model-events-5.10.2.jar
-kubernetes-model-extensions/5.10.2//kubernetes-model-extensions-5.10.2.jar
-kubernetes-model-flowcontrol/5.10.2//kubernetes-model-flowcontrol-5.10.2.jar
-kubernetes-model-metrics/5.10.2//kubernetes-model-metrics-5.10.2.jar
-kubernetes-model-networking/5.10.2//kubernetes-model-networking-5.10.2.jar
-kubernetes-model-node/5.10.2//kubernetes-model-node-5.10.2.jar
-kubernetes-model-policy/5.10.2//kubernetes-model-policy-5.10.2.jar
-kubernetes-model-rbac/5.10.2//kubernetes-model-rbac-5.10.2.jar
-kubernetes-model-scheduling/5.10.2//kubernetes-model-scheduling-5.10.2.jar
-kubernetes-model-storageclass/5.10.2//kubernetes-model-storageclass-5.10.2.jar
+kubernetes-client/5.12.0//kubernetes-client-5.12.0.jar
+kubernetes-model-admissionregistration/5.12.0//kubernetes-model-admissionregistration-5.12.0.jar
+kubernetes-model-apiextensions/5.12.0//kubernetes-model-apiextensions-5.12.0.jar
+kubernetes-model-apps/5.12.0//kubernetes-model-apps-5.12.0.jar
+kubernetes-model-autoscaling/5.12.0//kubernetes-model-autoscaling-5.12.0.jar
+kubernetes-model-batch/5.12.0//kubernetes-model-batch-5.12.0.jar
+kubernetes-model-certificates/5.12.0//kubernetes-model-certificates-5.12.0.jar
+kubernetes-model-common/5.12.0//kubernetes-model-common-5.12.0.jar
+kubernetes-model-coordination/5.12.0//kubernetes-model-coordination-5.12.0.jar
+kubernetes-model-core/5.12.0//kubernetes-model-core-5.12.0.jar
+kubernetes-model-discovery/5.12.0//kubernetes-model-discovery-5.12.0.jar
+kubernetes-model-events/5.12.0//kubernetes-model-events-5.12.0.jar
+kubernetes-model-extensions/5.12.0//kubernetes-model-extensions-5.12.0.jar
+kubernetes-model-flowcontrol/5.12.0//kubernetes-model-flowcontrol-5.12.0.jar
+kubernetes-model-metrics/5.12.0//kubernetes-model-metrics-5.12.0.jar
+kubernetes-model-networking/5.12.0//kubernetes-model-networking-5.12.0.jar
+kubernetes-model-node/5.12.0//kubernetes-model-node-5.12.0.jar
+kubernetes-model-policy/5.12.0//kubernetes-model-policy-5.12.0.jar
+kubernetes-model-rbac/5.12.0//kubernetes-model-rbac-5.12.0.jar
+kubernetes-model-scheduling/5.12.0//kubernetes-model-scheduling-5.12.0.jar
+kubernetes-model-storageclass/5.12.0//kubernetes-model-storageclass-5.12.0.jar
lapack/2.2.1//lapack-2.2.1.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
libfb303/0.9.3//libfb303-0.9.3.jar
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3
b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 3a38d07..a79a71b 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -109,7 +109,7 @@ jackson-core/2.13.1//jackson-core-2.13.1.jar
jackson-databind/2.13.1//jackson-databind-2.13.1.jar
jackson-dataformat-cbor/2.13.1//jackson-dataformat-cbor-2.13.1.jar
jackson-dataformat-yaml/2.13.1//jackson-dataformat-yaml-2.13.1.jar
-jackson-datatype-jsr310/2.13.0//jackson-datatype-jsr310-2.13.0.jar
+jackson-datatype-jsr310/2.13.1//jackson-datatype-jsr310-2.13.1.jar
jackson-mapper-asl/1.9.13//jackson-mapper-asl-1.9.13.jar
jackson-module-scala_2.12/2.13.1//jackson-module-scala_2.12-2.13.1.jar
jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar
@@ -148,27 +148,27 @@ jsr305/3.0.0//jsr305-3.0.0.jar
jta/1.1//jta-1.1.jar
jul-to-slf4j/1.7.32//jul-to-slf4j-1.7.32.jar
kryo-shaded/4.0.2//kryo-shaded-4.0.2.jar
-kubernetes-client/5.10.2//kubernetes-client-5.10.2.jar
-kubernetes-model-admissionregistration/5.10.2//kubernetes-model-admissionregistration-5.10.2.jar
-kubernetes-model-apiextensions/5.10.2//kubernetes-model-apiextensions-5.10.2.jar
-kubernetes-model-apps/5.10.2//kubernetes-model-apps-5.10.2.jar
-kubernetes-model-autoscaling/5.10.2//kubernetes-model-autoscaling-5.10.2.jar
-kubernetes-model-batch/5.10.2//kubernetes-model-batch-5.10.2.jar
-kubernetes-model-certificates/5.10.2//kubernetes-model-certificates-5.10.2.jar
-kubernetes-model-common/5.10.2//kubernetes-model-common-5.10.2.jar
-kubernetes-model-coordination/5.10.2//kubernetes-model-coordination-5.10.2.jar
-kubernetes-model-core/5.10.2//kubernetes-model-core-5.10.2.jar
-kubernetes-model-discovery/5.10.2//kubernetes-model-discovery-5.10.2.jar
-kubernetes-model-events/5.10.2//kubernetes-model-events-5.10.2.jar
-kubernetes-model-extensions/5.10.2//kubernetes-model-extensions-5.10.2.jar
-kubernetes-model-flowcontrol/5.10.2//kubernetes-model-flowcontrol-5.10.2.jar
-kubernetes-model-metrics/5.10.2//kubernetes-model-metrics-5.10.2.jar
-kubernetes-model-networking/5.10.2//kubernetes-model-networking-5.10.2.jar
-kubernetes-model-node/5.10.2//kubernetes-model-node-5.10.2.jar
-kubernetes-model-policy/5.10.2//kubernetes-model-policy-5.10.2.jar
-kubernetes-model-rbac/5.10.2//kubernetes-model-rbac-5.10.2.jar
-kubernetes-model-scheduling/5.10.2//kubernetes-model-scheduling-5.10.2.jar
-kubernetes-model-storageclass/5.10.2//kubernetes-model-storageclass-5.10.2.jar
+kubernetes-client/5.12.0//kubernetes-client-5.12.0.jar
+kubernetes-model-admissionregistration/5.12.0//kubernetes-model-admissionregistration-5.12.0.jar
+kubernetes-model-apiextensions/5.12.0//kubernetes-model-apiextensions-5.12.0.jar
+kubernetes-model-apps/5.12.0//kubernetes-model-apps-5.12.0.jar
+kubernetes-model-autoscaling/5.12.0//kubernetes-model-autoscaling-5.12.0.jar
+kubernetes-model-batch/5.12.0//kubernetes-model-batch-5.12.0.jar
+kubernetes-model-certificates/5.12.0//kubernetes-model-certificates-5.12.0.jar
+kubernetes-model-common/5.12.0//kubernetes-model-common-5.12.0.jar
+kubernetes-model-coordination/5.12.0//kubernetes-model-coordination-5.12.0.jar
+kubernetes-model-core/5.12.0//kubernetes-model-core-5.12.0.jar
+kubernetes-model-discovery/5.12.0//kubernetes-model-discovery-5.12.0.jar
+kubernetes-model-events/5.12.0//kubernetes-model-events-5.12.0.jar
+kubernetes-model-extensions/5.12.0//kubernetes-model-extensions-5.12.0.jar
+kubernetes-model-flowcontrol/5.12.0//kubernetes-model-flowcontrol-5.12.0.jar
+kubernetes-model-metrics/5.12.0//kubernetes-model-metrics-5.12.0.jar
+kubernetes-model-networking/5.12.0//kubernetes-model-networking-5.12.0.jar
+kubernetes-model-node/5.12.0//kubernetes-model-node-5.12.0.jar
+kubernetes-model-policy/5.12.0//kubernetes-model-policy-5.12.0.jar
+kubernetes-model-rbac/5.12.0//kubernetes-model-rbac-5.12.0.jar
+kubernetes-model-scheduling/5.12.0//kubernetes-model-scheduling-5.12.0.jar
+kubernetes-model-storageclass/5.12.0//kubernetes-model-storageclass-5.12.0.jar
lapack/2.2.1//lapack-2.2.1.jar
leveldbjni-all/1.8//leveldbjni-all-1.8.jar
libfb303/0.9.3//libfb303-0.9.3.jar
diff --git a/pom.xml b/pom.xml
index 9e18ba6..5bae4d2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -204,7 +204,7 @@
<arrow.version>6.0.1</arrow.version>
<!-- org.fusesource.leveldbjni will be used except on arm64 platform. -->
<leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
- <kubernetes-client.version>5.10.2</kubernetes-client.version>
+ <kubernetes-client.version>5.12.0</kubernetes-client.version>
<test.java.home>${java.home}</test.java.home>
diff --git
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
index 4131605..54f557c 100644
---
a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
+++
b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesClientFactory.scala
@@ -24,9 +24,10 @@ import com.google.common.io.Files
import io.fabric8.kubernetes.client.{ConfigBuilder, DefaultKubernetesClient,
KubernetesClient}
import
io.fabric8.kubernetes.client.Config.KUBERNETES_REQUEST_RETRY_BACKOFFLIMIT_SYSTEM_PROPERTY
import io.fabric8.kubernetes.client.Config.autoConfigure
-import io.fabric8.kubernetes.client.utils.HttpClientUtils
+import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory
import io.fabric8.kubernetes.client.utils.Utils.getSystemPropertyOrEnvVar
import okhttp3.Dispatcher
+import okhttp3.OkHttpClient
import org.apache.spark.SparkConf
import org.apache.spark.deploy.k8s.Config._
@@ -68,6 +69,8 @@ private[spark] object SparkKubernetesClientFactory extends
Logging {
.getOption(s"$kubernetesAuthConfPrefix.$CLIENT_KEY_FILE_CONF_SUFFIX")
val clientCertFile = sparkConf
.getOption(s"$kubernetesAuthConfPrefix.$CLIENT_CERT_FILE_CONF_SUFFIX")
+ // TODO(SPARK-37687): clean up direct usage of OkHttpClient, see also:
+ // https://github.com/fabric8io/kubernetes-client/issues/3547
val dispatcher = new Dispatcher(
ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher"))
@@ -105,13 +108,14 @@ private[spark] object SparkKubernetesClientFactory
extends Logging {
}.withOption(namespace) {
(ns, configBuilder) => configBuilder.withNamespace(ns)
}.build()
- val baseHttpClient = HttpClientUtils.createHttpClient(config)
- val httpClientWithCustomDispatcher = baseHttpClient.newBuilder()
- .dispatcher(dispatcher)
- .build()
+ val factoryWithCustomDispatcher = new OkHttpClientFactory() {
+ override protected def additionalConfig(builder: OkHttpClient.Builder):
Unit = {
+ builder.dispatcher(dispatcher)
+ }
+ }
logDebug("Kubernetes client config: " +
new
ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(config))
- new DefaultKubernetesClient(httpClientWithCustomDispatcher, config)
+ new
DefaultKubernetesClient(factoryWithCustomDispatcher.createHttpClient(config),
config)
}
private implicit class OptionConfigurableConfigBuilder(val configBuilder:
ConfigBuilder)
diff --git
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
index cc25853..e0fd926 100644
---
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
+++
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/Utils.scala
@@ -24,7 +24,7 @@ import java.util.zip.{ZipEntry, ZipOutputStream}
import scala.collection.JavaConverters._
import io.fabric8.kubernetes.client.dsl.ExecListener
-import okhttp3.Response
+import io.fabric8.kubernetes.client.dsl.ExecListener.Response
import org.apache.commons.compress.archivers.tar.{TarArchiveEntry,
TarArchiveOutputStream}
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream
import org.apache.commons.compress.utils.IOUtils
@@ -62,7 +62,7 @@ object Utils extends Logging {
val openLatch: CountDownLatch = new CountDownLatch(1)
val closeLatch: CountDownLatch = new CountDownLatch(1)
- override def onOpen(response: Response): Unit = {
+ override def onOpen(): Unit = {
openLatch.countDown()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
