Github user felixcheung commented on a diff in the pull request:
https://github.com/apache/spark/pull/19717#discussion_r154503142
--- Diff:
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
---
@@ -119,5 +139,60 @@ private[spark] object Config extends Logging {
"must be a positive integer")
.createWithDefault(10)
+ val WAIT_FOR_APP_COMPLETION =
+ ConfigBuilder("spark.kubernetes.submission.waitAppCompletion")
+ .doc("In cluster mode, whether to wait for the application to finish
before exiting the " +
+ "launcher process.")
+ .booleanConf
+ .createWithDefault(true)
+
+ val REPORT_INTERVAL =
+ ConfigBuilder("spark.kubernetes.report.interval")
+ .doc("Interval between reports of the current app status in cluster
mode.")
+ .timeConf(TimeUnit.MILLISECONDS)
+ .createWithDefaultString("1s")
+
+ private[spark] val JARS_DOWNLOAD_LOCATION =
+ ConfigBuilder("spark.kubernetes.mountDependencies.jarsDownloadDir")
+ .doc("Location to download jars to in the driver and executors. When
using" +
+ " spark-submit, this directory must be empty and will be mounted
as an empty directory" +
+ " volume on the driver and executor pod.")
+ .stringConf
+ .createWithDefault("/var/spark-data/spark-jars")
+
+ private[spark] val FILES_DOWNLOAD_LOCATION =
+ ConfigBuilder("spark.kubernetes.mountDependencies.filesDownloadDir")
+ .doc("Location to download files to in the driver and executors.
When using" +
+ " spark-submit, this directory must be empty and will be mounted
as an empty directory" +
+ " volume on the driver and executor pods.")
+ .stringConf
+ .createWithDefault("/var/spark-data/spark-files")
+
+ val KUBERNETES_AUTH_SUBMISSION_CONF_PREFIX =
+ "spark.kubernetes.authenticate.submission"
+
val KUBERNETES_NODE_SELECTOR_PREFIX = "spark.kubernetes.node.selector."
+
+ val KUBERNETES_DRIVER_LABEL_PREFIX = "spark.kubernetes.driver.label."
+ val KUBERNETES_DRIVER_ANNOTATION_PREFIX =
"spark.kubernetes.driver.annotation."
+
+ val KUBERNETES_EXECUTOR_LABEL_PREFIX = "spark.kubernetes.executor.label."
+ val KUBERNETES_EXECUTOR_ANNOTATION_PREFIX =
"spark.kubernetes.executor.annotation."
+
+ val KUBERNETES_DRIVER_ENV_KEY = "spark.kubernetes.driverEnv."
+
+ def getK8sMasterUrl(rawMasterString: String): String = {
+ require(rawMasterString.startsWith("k8s://"),
+ "Master URL should start with k8s:// in Kubernetes mode.")
+ val masterWithoutK8sPrefix = rawMasterString.substring("k8s://".length)
+ if (masterWithoutK8sPrefix.startsWith("http://";)
--- End diff --
btw, on a side note, should there be a log/warning for http: (vs https:)? I
don't think we have that case for other cluster manager
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
