Github user liyinan926 commented on a diff in the pull request:
https://github.com/apache/spark/pull/19954#discussion_r158651930
--- Diff:
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestrator.scala
---
@@ -98,28 +109,62 @@ private[spark] class
DriverConfigurationStepsOrchestrator(
None
}
- val sparkJars = submissionSparkConf.getOption("spark.jars")
+ val sparkJars = sparkConf.getOption("spark.jars")
.map(_.split(","))
.getOrElse(Array.empty[String]) ++
additionalMainAppJar.toSeq
- val sparkFiles = submissionSparkConf.getOption("spark.files")
+ val sparkFiles = sparkConf.getOption("spark.files")
.map(_.split(","))
.getOrElse(Array.empty[String])
- val maybeDependencyResolutionStep = if (sparkJars.nonEmpty ||
sparkFiles.nonEmpty) {
- Some(new DependencyResolutionStep(
+ val dependencyResolutionStep = if (sparkJars.nonEmpty ||
sparkFiles.nonEmpty) {
+ Seq(new DependencyResolutionStep(
sparkJars,
sparkFiles,
jarsDownloadPath,
filesDownloadPath))
} else {
- None
+ Nil
+ }
+
+ val initContainerBootstrapStep = if
(areAnyFilesNonContainerLocal(sparkJars ++ sparkFiles)) {
+ val orchestrator = new InitContainerConfigOrchestrator(
+ sparkJars,
+ sparkFiles,
+ jarsDownloadPath,
+ filesDownloadPath,
+ imagePullPolicy,
+ initContainerConfigMapName,
+ INIT_CONTAINER_PROPERTIES_FILE_NAME,
+ sparkConf)
+ val bootstrapStep = new DriverInitContainerBootstrapStep(
+ orchestrator.getAllConfigurationSteps,
+ initContainerConfigMapName,
+ INIT_CONTAINER_PROPERTIES_FILE_NAME)
+
+ Seq(bootstrapStep)
+ } else {
+ Nil
+ }
+
+ val mountSecretsStep = if (secretNamesToMountPaths.nonEmpty) {
+ Seq(new DriverMountSecretsStep(new
MountSecretsBootstrap(secretNamesToMountPaths)))
+ } else {
+ Nil
}
Seq(
initialSubmissionStep,
- driverAddressStep,
+ serviceBootstrapStep,
kubernetesCredentialsStep) ++
- maybeDependencyResolutionStep.toSeq
+ dependencyResolutionStep ++
+ initContainerBootstrapStep ++
+ mountSecretsStep
+ }
+
+ private def areAnyFilesNonContainerLocal(files: Seq[String]): Boolean = {
--- End diff --
Done.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
