Github user ueshin commented on a diff in the pull request:
https://github.com/apache/spark/pull/19954#discussion_r157248504
--- Diff:
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
---
@@ -45,6 +45,60 @@ private[spark] class KubernetesClusterManager extends
ExternalClusterManager wit
masterURL: String,
scheduler: TaskScheduler): SchedulerBackend = {
val sparkConf = sc.getConf
+ val maybeInitContainerConfigMap =
sparkConf.get(INIT_CONTAINER_CONFIG_MAP_NAME)
+ val maybeInitContainerConfigMapKey =
sparkConf.get(INIT_CONTAINER_CONFIG_MAP_KEY_CONF)
+
+ if (maybeInitContainerConfigMap.isEmpty) {
+ logWarning("The executor's init-container config map was not
specified. Executors will " +
+ "therefore not attempt to fetch remote or submitted dependencies.")
+ }
+
+ if (maybeInitContainerConfigMapKey.isEmpty) {
+ logWarning("The executor's init-container config map key was not
specified. Executors will " +
+ "therefore not attempt to fetch remote or submitted dependencies.")
+ }
+
+ // Only set up the bootstrap if they've provided both the config map
key and the config map
+ // name. The config map might not be provided if init-containers
aren't being used to
+ // bootstrap dependencies.
+ val maybeInitContainerBootstrap = for {
+ configMap <- maybeInitContainerConfigMap
+ configMapKey <- maybeInitContainerConfigMapKey
+ } yield {
+ val initContainerImage = sparkConf
+ .get(INIT_CONTAINER_DOCKER_IMAGE)
+ .getOrElse(throw new SparkException(
+ "Must specify the init-container Docker image when there are
remote dependencies"))
+ new InitContainerBootstrapImpl(
+ initContainerImage,
+ sparkConf.get(DOCKER_IMAGE_PULL_POLICY),
+ sparkConf.get(JARS_DOWNLOAD_LOCATION),
+ sparkConf.get(FILES_DOWNLOAD_LOCATION),
+ sparkConf.get(INIT_CONTAINER_MOUNT_TIMEOUT),
+ configMap,
+ configMapKey,
+ SPARK_POD_EXECUTOR_ROLE,
+ sparkConf)
+ }
+
+ val executorSecretNamesToMountPaths =
ConfigurationUtils.parsePrefixedKeyValuePairs(
+ sparkConf, KUBERNETES_EXECUTOR_SECRETS_PREFIX)
+ val mayBeMountSecretBootstrap = if
(executorSecretNamesToMountPaths.nonEmpty) {
+ Some(new MountSecretsBootstrapImpl(executorSecretNamesToMountPaths))
+ } else {
+ None
+ }
+ // Mount user-specified executor secrets also into the executor's
init-container. The
+ // init-container may need credentials in the secrets to be able to
download remote
+ // dependencies. The executor's main container and its init-container
share the secrets
+ // because the init-container is sort of an implementation details and
this sharing
+ // avoids introducing a dedicated configuration property just for the
init-container.
+ val mayBeInitContainerMountSecretsBootstrap = if
(maybeInitContainerBootstrap.nonEmpty &&
--- End diff --
Oh, I see. Thanks!
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
