Github user vanzin commented on a diff in the pull request:
https://github.com/apache/spark/pull/19954#discussion_r157322857
--- Diff:
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigurationStepsOrchestrator.scala
---
@@ -116,10 +122,53 @@ private[spark] class
DriverConfigurationStepsOrchestrator(
None
}
+ val mayBeInitContainerBootstrapStep =
+ if (areAnyFilesNonContainerLocal(sparkJars ++ sparkFiles)) {
+ val initContainerConfigurationStepsOrchestrator =
+ new InitContainerConfigurationStepsOrchestrator(
+ namespace,
+ kubernetesResourceNamePrefix,
+ sparkJars,
+ sparkFiles,
+ jarsDownloadPath,
+ filesDownloadPath,
+ dockerImagePullPolicy,
+ allDriverLabels,
+ initContainerConfigMapName,
+ INIT_CONTAINER_PROPERTIES_FILE_NAME,
+ submissionSparkConf)
+ val initContainerConfigurationSteps =
+
initContainerConfigurationStepsOrchestrator.getAllConfigurationSteps()
+ val initContainerBootstrapStep =
+ new DriverInitContainerBootstrapStep(
+ initContainerConfigurationSteps,
+ initContainerConfigMapName,
+ INIT_CONTAINER_PROPERTIES_FILE_NAME)
+
+ Some(initContainerBootstrapStep)
+ } else {
+ None
+ }
+
+ val mayBeMountSecretsStep = if
(driverSecretNamesToMountPaths.nonEmpty) {
+ val mountSecretsBootstrap = new
MountSecretsBootstrapImpl(driverSecretNamesToMountPaths)
+ Some(new DriverMountSecretsStep(mountSecretsBootstrap))
+ } else {
+ None
+ }
+
Seq(
initialSubmissionStep,
driverAddressStep,
kubernetesCredentialsStep) ++
- maybeDependencyResolutionStep.toSeq
+ maybeDependencyResolutionStep.toSeq ++
+ mayBeInitContainerBootstrapStep.toSeq ++
+ mayBeMountSecretsStep.toSeq
+ }
+
+ private def areAnyFilesNonContainerLocal(files: Seq[String]): Boolean = {
+ files.exists { uri =>
+ Option(Utils.resolveURI(uri).getScheme).getOrElse("file") != "local"
--- End diff --
`Utils.resolveURI(uri) != "local"`
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
