ruanwenjun commented on code in PR #53307:
URL: https://github.com/apache/spark/pull/53307#discussion_r2608851638
##########
docs/running-on-kubernetes.md:
##########
@@ -2011,6 +1973,17 @@ spec:
queue: default
```
+You have two options to provide the PodGroup template in spark.
Review Comment:
The `podGroupTemplateFile` will be used preferentially, I have documented
this.
##########
resource-managers/kubernetes/core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala:
##########
@@ -59,13 +66,42 @@ private[spark] class VolcanoFeatureStep extends
KubernetesDriverCustomFeatureCon
if (spec == null) spec = new PodGroupSpec
pg.setSpec(spec)
+ logDebug(s"Volcano PodGroup configuration: $pg")
Seq(pg)
}
+ private def getPodGroupConfigByTemplateFile(
+ volcanoClient: DefaultVolcanoClient): Option[PodGroup] = {
+ kubernetesConf.getOption(POD_GROUP_TEMPLATE_FILE_KEY).map { templateFile =>
+ logDebug("Loading Volcano PodGroup configuration from template file")
+ volcanoClient.podGroups.load(templateFile).item
+ }
+ }
+
+ private def getPodGroupByTemplateJson(volcanoClient: DefaultVolcanoClient):
Option[PodGroup] = {
+ kubernetesConf.getOption(POD_GROUP_TEMPLATE_JSON_KEY).map { templateJson =>
+ logDebug("Loading Volcano PodGroup configuration from template json")
+ try {
+ val templateYaml = new ObjectMapper()
+ .writerWithDefaultPrettyPrinter()
+ .writeValueAsString(new ObjectMapper(new
YAMLFactory()).readTree(templateJson))
+ volcanoClient
+ .podGroups()
+ .load(new java.io.ByteArrayInputStream(templateYaml.getBytes()))
Review Comment:
Done
##########
resource-managers/kubernetes/core/volcano/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala:
##########
@@ -59,13 +66,42 @@ private[spark] class VolcanoFeatureStep extends
KubernetesDriverCustomFeatureCon
if (spec == null) spec = new PodGroupSpec
pg.setSpec(spec)
+ logDebug(s"Volcano PodGroup configuration: $pg")
Seq(pg)
}
+ private def getPodGroupConfigByTemplateFile(
+ volcanoClient: DefaultVolcanoClient): Option[PodGroup] = {
+ kubernetesConf.getOption(POD_GROUP_TEMPLATE_FILE_KEY).map { templateFile =>
+ logDebug("Loading Volcano PodGroup configuration from template file")
+ volcanoClient.podGroups.load(templateFile).item
+ }
+ }
+
+ private def getPodGroupByTemplateJson(volcanoClient: DefaultVolcanoClient):
Option[PodGroup] = {
+ kubernetesConf.getOption(POD_GROUP_TEMPLATE_JSON_KEY).map { templateJson =>
+ logDebug("Loading Volcano PodGroup configuration from template json")
+ try {
+ val templateYaml = new ObjectMapper()
+ .writerWithDefaultPrettyPrinter()
+ .writeValueAsString(new ObjectMapper(new
YAMLFactory()).readTree(templateJson))
+ volcanoClient
+ .podGroups()
+ .load(new java.io.ByteArrayInputStream(templateYaml.getBytes()))
+ .item()
+ } catch {
+ case e: Exception =>
+ throw new SparkException(
+ f"The ${POD_GROUP_TEMPLATE_JSON_KEY} provided is not validated",
+ e)
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
