Github user felixcheung commented on a diff in the pull request:
https://github.com/apache/spark/pull/21748#discussion_r204636048
--- Diff:
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
---
@@ -35,26 +35,39 @@ private[spark] class KubernetesClusterManager extends
ExternalClusterManager wit
override def canCreate(masterURL: String): Boolean =
masterURL.startsWith("k8s")
override def createTaskScheduler(sc: SparkContext, masterURL: String):
TaskScheduler = {
- if (masterURL.startsWith("k8s") &&
- sc.deployMode == "client" &&
- !sc.conf.get(KUBERNETES_DRIVER_SUBMIT_CHECK).getOrElse(false)) {
- throw new SparkException("Client mode is currently not supported for
Kubernetes.")
- }
-
new TaskSchedulerImpl(sc)
}
override def createSchedulerBackend(
sc: SparkContext,
masterURL: String,
scheduler: TaskScheduler): SchedulerBackend = {
+ val wasSparkSubmittedInClusterMode =
sc.conf.get(KUBERNETES_DRIVER_SUBMIT_CHECK)
+ val (authConfPrefix,
+ apiServerUri,
+ defaultServiceAccountToken,
+ defaultServiceAccountCaCrt) = if (wasSparkSubmittedInClusterMode) {
+ require(sc.conf.get(KUBERNETES_DRIVER_POD_NAME).isDefined,
+ "If the application is deployed using spark-submit in cluster
mode, the driver pod name " +
+ "must be provided.")
+ (KUBERNETES_AUTH_DRIVER_MOUNTED_CONF_PREFIX,
+ KUBERNETES_MASTER_INTERNAL_URL,
+ Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH)),
+ Some(new File(Config.KUBERNETES_SERVICE_ACCOUNT_CA_CRT_PATH)))
+ } else {
+ (KUBERNETES_AUTH_CLIENT_MODE_PREFIX,
+ masterURL.substring("k8s://".length()),
--- End diff --
I thought there's some function for parsing the k8s master url?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
