This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 4e22642 SUBMARINE-709. Singleton tensorboard serving all experiments
4e22642 is described below
commit 4e22642478a29d1c98b388a9fd1d1c27fb6b5bae
Author: Byron <[email protected]>
AuthorDate: Fri Jan 29 15:43:57 2021 +0800
SUBMARINE-709. Singleton tensorboard serving all experiments
### What is this PR for?
In order to save the resource, we use a single tensorboard for all
experiments.
<img
src="https://user-images.githubusercontent.com/24364830/104589865-7657f300-56a5-11eb-969d-60d7f4567625.jpg";
width="500px">
The implementation details are demonstrated in this figure:
1. There is a PV for storing tensorborad log files, and a corresponding PVC.
2. There is a tensorboard pod serving all the logs, and it can be accessed
by the ingress service.
3. MLJob writes logs to PV, and tensorboard reads logs from PV.
4. PV, PVC, and tensorboard pod are created in helm install stage.
### What type of PR is it?
[Feature]
### Todos
* [ ] - Frontend support
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-709
### How should this be tested?
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Byron <[email protected]>
Signed-off-by: Kevin <[email protected]>
Closes #491 from ByronHsu/SINGLETON_TENSORBOARD and squashes the following
commits:
947af10 [Byron] fix import problem
f2cf399 [Byron] merge conflict
22204a7 [Byron] revise rbac in cloud
bd905e9 [Byron] fix style
56f2f9c [Byron] dont hard-code tensorboardinfo response in test
5177fc2 [Byron] fill the placeholder of yaml in submarine cloud
8866ced [Byron] add license
67b3d4a [Byron] add assert in test
5114e51 [Byron] add getTensorBoardInfo api
1b7e1e6 [Byron] delete unused files & add tensorboard to submarine cloud
6833a60 [Byron] add tensorboard helm template & revise MJjob parser
---
helm-charts/submarine/templates/rbac.yaml | 1 +
.../submarine/templates/submarine-tensorboard.yaml | 99 +++++++++++++++
helm-charts/submarine/values.yaml | 4 +
.../manifests/submarine-cluster/rbac.yaml | 1 +
.../submarine-cluster/submarine-tensorboard.yaml | 99 +++++++++++++++
.../org/apache/submarine/server/api/Submitter.java | 9 ++
.../server/api/experiment/Experiment.java | 9 --
.../server/api/experiment/TensorboardInfo.java | 54 ++++++++
.../server/experiment/ExperimentManager.java | 17 ++-
.../submarine/server/rest/ExperimentRestApi.java | 24 +++-
.../server/submitter/k8s/K8sSubmitter.java | 137 +++++++-------------
.../submitter/k8s/parser/ExperimentSpecParser.java | 8 +-
.../k8s/parser/TensorboardSpecParser.java | 141 ---------------------
.../submitter/k8s/parser/VolumeSpecParser.java | 1 -
.../submitter/k8s/util/TensorboardUtils.java | 38 ------
.../server/submitter/k8s/K8SJobSubmitterTest.java | 30 +++--
.../server/submitter/k8s/SpecBuilder.java | 2 +-
.../k8s/parser/TensorboardSpecParserTest.java | 80 ------------
.../submitter/k8s/parser/VolumeSpecParserTest.java | 64 ----------
19 files changed, 368 insertions(+), 450 deletions(-)
diff --git a/helm-charts/submarine/templates/rbac.yaml
b/helm-charts/submarine/templates/rbac.yaml
index bed8c19..1fc6526 100644
--- a/helm-charts/submarine/templates/rbac.yaml
+++ b/helm-charts/submarine/templates/rbac.yaml
@@ -65,6 +65,7 @@ rules:
- "apps"
resources:
- deployments
+ - deployments/status
verbs:
- '*'
---
diff --git a/helm-charts/submarine/templates/submarine-tensorboard.yaml
b/helm-charts/submarine/templates/submarine-tensorboard.yaml
new file mode 100644
index 0000000..ea71d00
--- /dev/null
+++ b/helm-charts/submarine/templates/submarine-tensorboard.yaml
@@ -0,0 +1,99 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: tensorboard-pv
+spec:
+ accessModes:
+ - ReadWriteMany
+ capacity:
+ storage: "{{ .Values.submarine.tensorboard.storage }}"
+ storageClassName: standard
+ hostPath:
+ path: "{{ .Values.submarine.tensorboard.hostPath }}"
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: tensorboard-pvc
+spec:
+ accessModes:
+ - ReadWriteMany
+ storageClassName: standard
+ resources:
+ requests:
+ storage: "{{ .Values.submarine.tensorboard.storage }}"
+ volumeName: tensorboard-pv # bind to specific pv
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: tensorboard
+spec:
+ selector:
+ matchLabels:
+ app: tensorboard-pod
+ template:
+ metadata:
+ labels:
+ app: tensorboard-pod
+ spec:
+ containers:
+ - name: tensorboard-container
+ image: tensorflow/tensorflow:1.11.0
+ command:
+ - "tensorboard"
+ - "--logdir=/logs"
+ - "--path_prefix=/tensorboard"
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 6006
+ volumeMounts:
+ - mountPath: "/logs"
+ name: "volume"
+ volumes:
+ - name: "volume"
+ persistentVolumeClaim:
+ claimName: "tensorboard-pvc"
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: tensorboard-service
+spec:
+ selector:
+ app: tensorboard-pod
+ ports:
+ - protocol: TCP
+ port: 8080
+ targetPort: 6006
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+ name: tensorboard-ingressroute
+spec:
+ entryPoints:
+ - web
+ routes:
+ - kind: Rule
+ match: "{{ .Values.submarine.tensorboard.ingressPath }}"
+ services:
+ - kind: Service
+ name: tensorboard-service
+ port: 8080
diff --git a/helm-charts/submarine/values.yaml
b/helm-charts/submarine/values.yaml
index e1f1600..2571ddd 100644
--- a/helm-charts/submarine/values.yaml
+++ b/helm-charts/submarine/values.yaml
@@ -31,3 +31,7 @@ submarine:
mysqlRootPassword: password
traefik:
enabled: true
+ tensorboard:
+ storage: 10Gi
+ hostPath: /tmp/tfboard/
+ ingressPath: PathPrefix(`/tensorboard`)
diff --git a/submarine-cloud/manifests/submarine-cluster/rbac.yaml
b/submarine-cloud/manifests/submarine-cluster/rbac.yaml
index 6232d3b..9c12225 100644
--- a/submarine-cloud/manifests/submarine-cluster/rbac.yaml
+++ b/submarine-cloud/manifests/submarine-cluster/rbac.yaml
@@ -34,6 +34,7 @@ items:
- apiGroups: ["apps"]
resources:
- deployments
+ - deployments/status
verbs: ["*"]
- apiGroups: ["kubeflow.org"]
resources:
diff --git
a/submarine-cloud/manifests/submarine-cluster/submarine-tensorboard.yaml
b/submarine-cloud/manifests/submarine-cluster/submarine-tensorboard.yaml
new file mode 100644
index 0000000..c1aa336
--- /dev/null
+++ b/submarine-cloud/manifests/submarine-cluster/submarine-tensorboard.yaml
@@ -0,0 +1,99 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: tensorboard-pv
+spec:
+ accessModes:
+ - ReadWriteMany
+ capacity:
+ storage: 10Gi
+ storageClassName: standard
+ hostPath:
+ path: /tmp/tfboard/
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: tensorboard-pvc
+spec:
+ accessModes:
+ - ReadWriteMany
+ storageClassName: standard
+ resources:
+ requests:
+ storage: 10Gi
+ volumeName: tensorboard-pv # bind to specific pv
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: tensorboard
+spec:
+ selector:
+ matchLabels:
+ app: tensorboard-pod
+ template:
+ metadata:
+ labels:
+ app: tensorboard-pod
+ spec:
+ containers:
+ - name: tensorboard-container
+ image: tensorflow/tensorflow:1.11.0
+ command:
+ - "tensorboard"
+ - "--logdir=/logs"
+ - "--path_prefix=/tensorboard"
+ imagePullPolicy: IfNotPresent
+ ports:
+ - containerPort: 6006
+ volumeMounts:
+ - mountPath: "/logs"
+ name: "volume"
+ volumes:
+ - name: "volume"
+ persistentVolumeClaim:
+ claimName: "tensorboard-pvc"
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: tensorboard-service
+spec:
+ selector:
+ app: tensorboard-pod
+ ports:
+ - protocol: TCP
+ port: 8080
+ targetPort: 6006
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+ name: tensorboard-ingressroute
+spec:
+ entryPoints:
+ - web
+ routes:
+ - kind: Rule
+ match: PathPrefix(`/tensorboard`)
+ services:
+ - kind: Service
+ name: tensorboard-service
+ port: 8080
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
index d3fd2e9..c52450e 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
@@ -23,6 +23,7 @@ import
org.apache.submarine.commons.utils.SubmarineConfiguration;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.ExperimentLog;
+import org.apache.submarine.server.api.experiment.TensorboardInfo;
import org.apache.submarine.server.api.notebook.Notebook;
import org.apache.submarine.server.api.spec.ExperimentSpec;
import org.apache.submarine.server.api.spec.NotebookSpec;
@@ -119,4 +120,12 @@ public interface Submitter {
* @throws SubmarineRuntimeException running error
*/
List<Notebook> listNotebook(String id) throws SubmarineRuntimeException;
+
+ /**
+ * Get tensorboard meta data
+ * @param
+ * @return object
+ * @throws SubmarineRuntimeException running error
+ */
+ TensorboardInfo getTensorboardInfo() throws SubmarineRuntimeException;
}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/Experiment.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/Experiment.java
index 932b1a6..2f14460 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/Experiment.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/Experiment.java
@@ -34,7 +34,6 @@ public class Experiment {
private String runningTime;
private String finishedTime;
private ExperimentSpec spec;
- private String tfboardURL;
/**
* Get the job id which is unique in submarine
@@ -136,14 +135,6 @@ public class Experiment {
this.spec = spec;
}
- public String getTfboardURL() {
- return tfboardURL;
- }
-
- public void setTfboardURL(String tfboardURL) {
- this.tfboardURL = tfboardURL;
- }
-
public enum Status {
STATUS_ACCEPTED("Accepted"),
STATUS_CREATED("Created"),
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/TensorboardInfo.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/TensorboardInfo.java
new file mode 100644
index 0000000..2c29ab7
--- /dev/null
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/TensorboardInfo.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.api.experiment;
+
+public class TensorboardInfo {
+ public boolean available;
+ public String url;
+
+ public TensorboardInfo(boolean available, String url) {
+ this.available = available;
+ this.url = url;
+ }
+
+ public boolean isAvailable() {
+ return available;
+ }
+
+ public void setAvailable(boolean available) {
+ this.available = available;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ @Override
+ public String toString() {
+ return "TensorboardInfo{" +
+ "available=" + available +
+ ", url='" + url + '\'' +
+ '}';
+ }
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
index 6da31db..92c8da9 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
@@ -36,6 +36,7 @@ import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.ExperimentId;
import org.apache.submarine.server.api.Submitter;
import org.apache.submarine.server.api.experiment.ExperimentLog;
+import org.apache.submarine.server.api.experiment.TensorboardInfo;
import org.apache.submarine.server.api.spec.ExperimentSpec;
import org.apache.submarine.server.rest.RestConstants;
import org.slf4j.Logger;
@@ -101,12 +102,6 @@ public class ExperimentManager {
Experiment experiment = submitter.createExperiment(spec);
experiment.setExperimentId(id);
- /*
- TODO(byronhsu) Importing tensorboardUtils will
- cause a dependency circle. Hard-code it as a temporary solution
- */
-
- experiment.setTfboardURL("/tfboard-" + spec.getMeta().getName() + "/");
spec.getMeta().getEnvVars().remove(RestConstants.JOB_ID);
spec.getMeta().getEnvVars().remove(RestConstants.SUBMARINE_TRACKING_URI);
@@ -241,6 +236,16 @@ public class ExperimentManager {
return submitter.getExperimentLog(spec, id);
}
+ /**
+ * Get tensorboard meta data
+ *
+ * @return tensorboardinfo
+ * @throws SubmarineRuntimeException the service error
+ */
+ public TensorboardInfo getTensorboardInfo() throws SubmarineRuntimeException
{
+ return submitter.getTensorboardInfo();
+ }
+
private void checkSpec(ExperimentSpec spec) throws SubmarineRuntimeException
{
if (spec == null) {
throw new SubmarineRuntimeException(Status.OK.getStatusCode(), "Invalid
experiment spec.");
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java
index a7f8973..c3dc873 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java
@@ -41,6 +41,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.server.api.experiment.Experiment;
+import org.apache.submarine.server.api.experiment.TensorboardInfo;
import org.apache.submarine.server.experiment.ExperimentManager;
import
org.apache.submarine.server.experimenttemplate.ExperimentTemplateManager;
import org.apache.submarine.server.api.experiment.ExperimentLog;
@@ -108,7 +109,7 @@ public class ExperimentRestApi {
/**
* Returns the contents of {@link Experiment} that submitted by user.
*
- * @param id template id
+ * @param name template name
* @param spec
* @return the contents of experiment
*/
@@ -227,7 +228,8 @@ public class ExperimentRestApi {
tags = {"experiment"},
responses = {
@ApiResponse(description = "successful operation", content =
@Content(
- schema = @Schema(implementation = JsonResponse.class)))})
+ schema = @Schema(implementation = JsonResponse.class))),
+ })
public Response listLog(@QueryParam("status") String status) {
try {
List<ExperimentLog> experimentLogList =
experimentManager.listExperimentLogsByStatus(status);
@@ -258,6 +260,24 @@ public class ExperimentRestApi {
}
}
+ @GET
+ @Path("/tensorboard")
+ @Operation(summary = "Get tensorboard's information",
+ tags = {"experiment"},
+ responses = {
+ @ApiResponse(description = "successful operation", content = @Content(
+ schema = @Schema(implementation = JsonResponse.class))),
+ @ApiResponse(responseCode = "404", description = "Tensorboard not
found")})
+ public Response getTensorboardInfo() {
+ try {
+ TensorboardInfo tensorboardInfo = experimentManager.getTensorboardInfo();
+ return new
JsonResponse.Builder<TensorboardInfo>(Response.Status.OK).success(true)
+ .result(tensorboardInfo).build();
+ } catch (SubmarineRuntimeException e) {
+ return parseExperimentServiceException(e);
+ }
+ }
+
private Response parseExperimentServiceException(SubmarineRuntimeException
e) {
return new JsonResponse.Builder<String>(e.getCode())
.message(e.getMessage().equals("Conflict") ? "Duplicated experiment
name" : e.getMessage()).build();
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
index 6d5b50d..3935acf 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
@@ -23,9 +23,10 @@ import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.List;
+
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import io.kubernetes.client.ApiClient;
@@ -40,10 +41,9 @@ import io.kubernetes.client.models.V1Deployment;
import io.kubernetes.client.models.V1ObjectMeta;
import io.kubernetes.client.models.V1PersistentVolume;
import io.kubernetes.client.models.V1PersistentVolumeClaim;
+import io.kubernetes.client.models.V1PersistentVolumeClaimVolumeSource;
import io.kubernetes.client.models.V1Pod;
import io.kubernetes.client.models.V1PodList;
-import io.kubernetes.client.models.V1Service;
-import io.kubernetes.client.models.V1PersistentVolumeClaimVolumeSource;
import io.kubernetes.client.models.V1Status;
import io.kubernetes.client.util.ClientBuilder;
import io.kubernetes.client.util.KubeConfig;
@@ -53,6 +53,7 @@ import org.apache.submarine.server.api.Submitter;
import org.apache.submarine.server.api.exception.InvalidSpecException;
import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.ExperimentLog;
+import org.apache.submarine.server.api.experiment.TensorboardInfo;
import org.apache.submarine.server.api.notebook.Notebook;
import org.apache.submarine.server.api.spec.ExperimentMeta;
import org.apache.submarine.server.api.spec.ExperimentSpec;
@@ -64,11 +65,9 @@ import
org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRoute
import org.apache.submarine.server.submitter.k8s.model.ingressroute.SpecRoute;
import org.apache.submarine.server.submitter.k8s.parser.ExperimentSpecParser;
import org.apache.submarine.server.submitter.k8s.parser.NotebookSpecParser;
-import org.apache.submarine.server.submitter.k8s.parser.TensorboardSpecParser;
import org.apache.submarine.server.submitter.k8s.parser.VolumeSpecParser;
import org.apache.submarine.server.submitter.k8s.util.MLJobConverter;
import org.apache.submarine.server.submitter.k8s.util.NotebookUtils;
-import org.apache.submarine.server.submitter.k8s.util.TensorboardUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,12 +90,11 @@ public class K8sSubmitter implements Submitter {
private AppsV1Api appsV1Api;
- public K8sSubmitter() { }
+ public K8sSubmitter() {}
@Override
public void initialize(SubmarineConfiguration conf) {
ApiClient client = null;
-
try {
String path = System.getenv(KUBECONFIG_ENV);
KubeConfig config = KubeConfig.loadKubeConfig(new FileReader(path));
@@ -119,30 +117,19 @@ public class K8sSubmitter implements Submitter {
if (coreApi == null) {
coreApi = new CoreV1Api(client);
}
+
if (appsV1Api == null) {
appsV1Api = new AppsV1Api();
}
- client.setDebugging(true);
+ // client.setDebugging(true);
}
@Override
public Experiment createExperiment(ExperimentSpec spec) throws
SubmarineRuntimeException {
Experiment experiment;
- final String name = spec.getMeta().getName();
- final String pvName = TensorboardUtils.PV_PREFIX + name;
- final String hostPath = TensorboardUtils.HOST_PREFIX + name;
- final String storage = TensorboardUtils.STORAGE;
- final String pvcName = TensorboardUtils.PVC_PREFIX + name;
- final String volume = TensorboardUtils.PV_PREFIX + name;
-
try {
MLJob mlJob = ExperimentSpecParser.parseJob(spec);
-
- createPersistentVolume(pvName, hostPath, storage);
- createPersistentVolumeClaim(pvcName, spec.getMeta().getNamespace(),
volume, storage);
- createTFBoard(name, spec.getMeta().getNamespace());
-
Object object = api.createNamespacedCustomObject(mlJob.getGroup(),
mlJob.getVersion(),
mlJob.getMetadata().getNamespace(), mlJob.getPlural(), mlJob,
"true");
experiment = parseExperimentResponseObject(object,
ParseOp.PARSE_OP_RESULT);
@@ -193,17 +180,8 @@ public class K8sSubmitter implements Submitter {
@Override
public Experiment deleteExperiment(ExperimentSpec spec) throws
SubmarineRuntimeException {
Experiment experiment;
- final String name = spec.getMeta().getName(); //
spec.getMeta().getEnvVars().get(RestConstants.JOB_ID);
- final String pvName = TensorboardUtils.PV_PREFIX + name;
- final String pvcName = TensorboardUtils.PVC_PREFIX + name;
-
try {
MLJob mlJob = ExperimentSpecParser.parseJob(spec);
-
- deletePersistentVolume(pvName);
- deletePersistentVolumeClaim(pvcName, spec.getMeta().getNamespace());
- deleteTFBoard(name, spec.getMeta().getNamespace());
-
Object object = api.deleteNamespacedCustomObject(mlJob.getGroup(),
mlJob.getVersion(),
mlJob.getMetadata().getNamespace(), mlJob.getPlural(),
mlJob.getMetadata().getName(),
MLJobConverter.toDeleteOptionsFromMLJob(mlJob), null, null, null);
@@ -283,6 +261,45 @@ public class K8sSubmitter implements Submitter {
}
@Override
+ public TensorboardInfo getTensorboardInfo() throws SubmarineRuntimeException
{
+ final String name = "tensorboard";
+ final String namespace = "default";
+ final String ingressRouteName = "tensorboard-ingressroute";
+
+ try {
+ V1Deployment deploy = appsV1Api.readNamespacedDeploymentStatus(name,
namespace, "true");
+ boolean available = deploy.getStatus().getAvailableReplicas() > 0; // at
least one replica is running
+
+ IngressRoute ingressRoute = new IngressRoute();
+ V1ObjectMeta meta = new V1ObjectMeta();
+ meta.setName(ingressRouteName);
+ meta.setNamespace(namespace);
+ ingressRoute.setMetadata(meta);
+ Object object = api.getNamespacedCustomObject(
+ ingressRoute.getGroup(), ingressRoute.getVersion(),
+ ingressRoute.getMetadata().getNamespace(),
+ ingressRoute.getPlural(), ingressRouteName
+ );
+
+ Gson gson = new JSON().getGson();
+ String jsonString = gson.toJson(object);
+ IngressRoute result = gson.fromJson(jsonString, IngressRoute.class);
+
+
+ String route =
result.getSpec().getRoutes().stream().findFirst().get().getMatch();
+
+ // replace "PathPrefix(`/tensorboard`)" with "/tensorboard/"
+ String url = route.replace("PathPrefix(`", "").replace("`)", "/");
+
+ TensorboardInfo tensorboardInfo = new TensorboardInfo(available, url);
+
+ return tensorboardInfo;
+ } catch (ApiException e) {
+ throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
+ }
+ }
+
+ @Override
public Notebook createNotebook(NotebookSpec spec) throws
SubmarineRuntimeException {
Notebook notebook;
final String name = spec.getMeta().getName();
@@ -379,68 +396,6 @@ public class K8sSubmitter implements Submitter {
return notebookList;
}
- public void createTFBoard(String name, String namespace) throws ApiException
{
- final String deployName = TensorboardUtils.DEPLOY_PREFIX + name;
- final String podName = TensorboardUtils.POD_PREFIX + name;
- final String svcName = TensorboardUtils.SVC_PREFIX + name;
- final String ingressName = TensorboardUtils.INGRESS_PREFIX + name;
-
- final String image = TensorboardUtils.IMAGE_NAME;
- final String routePath = TensorboardUtils.PATH_PREFIX + name;
- final String pvc = TensorboardUtils.PVC_PREFIX + name;
-
- V1Deployment deployment =
TensorboardSpecParser.parseDeployment(deployName, image, routePath, pvc);
- V1Service svc = TensorboardSpecParser.parseService(svcName, podName);
- IngressRoute ingressRoute = TensorboardSpecParser.parseIngressRoute(
- ingressName, namespace, routePath, svcName
- );
-
- try {
- appsV1Api.createNamespacedDeployment(namespace, deployment, "true",
null, null);
- coreApi.createNamespacedService(namespace, svc, "true", null, null);
- api.createNamespacedCustomObject(
- ingressRoute.getGroup(), ingressRoute.getVersion(),
- ingressRoute.getMetadata().getNamespace(),
- ingressRoute.getPlural(), ingressRoute, "true");
- } catch (ApiException e) {
- LOG.error("Exception when creating TensorBoard " + e.getMessage(), e);
- throw e;
- }
- }
-
- public void deleteTFBoard(String name, String namespace) throws ApiException
{
- final String deployName = TensorboardUtils.DEPLOY_PREFIX + name;
- final String podName = TensorboardUtils.POD_PREFIX + name;
- final String svcName = TensorboardUtils.SVC_PREFIX + name;
- final String ingressName = TensorboardUtils.INGRESS_PREFIX + name;
-
- final String image = TensorboardUtils.IMAGE_NAME;
- final String routePath = TensorboardUtils.PATH_PREFIX + name;
- final String pvc = TensorboardUtils.PVC_PREFIX + name;
-
- V1Deployment deployment =
TensorboardSpecParser.parseDeployment(deployName, image, routePath, pvc);
- V1Service svc = TensorboardSpecParser.parseService(svcName, podName);
- IngressRoute ingressRoute = TensorboardSpecParser.parseIngressRoute(
- ingressName, namespace, routePath, svcName
- );
-
- try {
- appsV1Api.deleteNamespacedDeployment(deployName, namespace, "true",
- null, null, null, null, null);
- coreApi.deleteNamespacedService(svcName, namespace, "true",
- null, null, null, null, null);
- api.deleteNamespacedCustomObject(
- ingressRoute.getGroup(), ingressRoute.getVersion(),
- ingressRoute.getMetadata().getNamespace(), ingressRoute.getPlural(),
ingressName,
- new
V1DeleteOptionsBuilder().withApiVersion(ingressRoute.getApiVersion()).build(),
- null, null, null);
-
- } catch (ApiException e) {
- LOG.error("Exception when deleting TensorBoard " + e.getMessage(), e);
- throw e;
- }
- }
-
public void createPersistentVolume(String pvName, String hostPath, String
storage) throws ApiException {
V1PersistentVolume pv = VolumeSpecParser.parsePersistentVolume(pvName,
hostPath, storage);
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
index 29e3b0b..8b8df40 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/ExperimentSpecParser.java
@@ -53,7 +53,6 @@ import
org.apache.submarine.server.submitter.k8s.model.pytorchjob.PyTorchJobSpec
import org.apache.submarine.server.submitter.k8s.model.tfjob.TFJob;
import org.apache.submarine.server.submitter.k8s.model.tfjob.TFJobReplicaType;
import org.apache.submarine.server.submitter.k8s.model.tfjob.TFJobSpec;
-import org.apache.submarine.server.submitter.k8s.util.TensorboardUtils;
import java.util.ArrayList;
import java.util.Arrays;
@@ -178,14 +177,15 @@ public class ExperimentSpecParser {
container.setResources(resources);
container.setEnv(parseEnvVars(taskSpec,
experimentSpec.getMeta().getEnvVars()));
+
+ final String name = experimentSpec.getMeta().getName();
// volumeMount
- container.addVolumeMountsItem(new
V1VolumeMount().mountPath("/logs").name("volume"));
+ container.addVolumeMountsItem(new
V1VolumeMount().mountPath("/logs").name("volume").subPath(name));
// volume
- final String name = experimentSpec.getMeta().getName();
V1Volume podVolume = new V1Volume().name("volume");
podVolume.setPersistentVolumeClaim(
- new
V1PersistentVolumeClaimVolumeSource().claimName(TensorboardUtils.PVC_PREFIX +
name)
+ new
V1PersistentVolumeClaimVolumeSource().claimName("tensorboard-pvc")
);
podSpec.addVolumesItem(podVolume);
/**
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/TensorboardSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/TensorboardSpecParser.java
deleted file mode 100644
index dfba33f..0000000
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/TensorboardSpecParser.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.submarine.server.submitter.k8s.parser;
-
-import io.kubernetes.client.custom.IntOrString;
-import io.kubernetes.client.models.V1Container;
-import io.kubernetes.client.models.V1ContainerPort;
-import io.kubernetes.client.models.V1Deployment;
-import io.kubernetes.client.models.V1DeploymentSpec;
-import io.kubernetes.client.models.V1LabelSelector;
-import io.kubernetes.client.models.V1ObjectMeta;
-import io.kubernetes.client.models.V1PersistentVolumeClaimVolumeSource;
-import io.kubernetes.client.models.V1PodSpec;
-import io.kubernetes.client.models.V1PodTemplateSpec;
-import io.kubernetes.client.models.V1Service;
-import io.kubernetes.client.models.V1ServicePort;
-import io.kubernetes.client.models.V1ServiceSpec;
-import io.kubernetes.client.models.V1Volume;
-import io.kubernetes.client.models.V1VolumeMount;
-import
org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRoute;
-import
org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRouteSpec;
-import org.apache.submarine.server.submitter.k8s.model.ingressroute.SpecRoute;
-import org.apache.submarine.server.submitter.k8s.util.TensorboardUtils;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-
-public class TensorboardSpecParser {
- public static V1Deployment parseDeployment(String name, String image, String
routePath, String pvc) {
-
- V1Deployment deployment = new V1Deployment();
-
- V1ObjectMeta deploymentMetedata = new V1ObjectMeta();
- deploymentMetedata.setName(name);
- deployment.setMetadata(deploymentMetedata);
- V1DeploymentSpec deploymentSpec = new V1DeploymentSpec();
- deploymentSpec.setSelector(
- new V1LabelSelector().matchLabels(Collections.singletonMap("app",
name)) // match the template
- );
-
- V1PodTemplateSpec deploymentTemplateSpec = new V1PodTemplateSpec();
- deploymentTemplateSpec.setMetadata(
- new V1ObjectMeta().labels(Collections.singletonMap("app", name)) //
bind to replicaset and service
- );
-
- V1PodSpec deploymentTemplatePodSpec = new V1PodSpec();
-
- V1Container container = new V1Container();
- container.setName(name);
- container.setImage(image);
- container.setCommand(Arrays.asList(
- "tensorboard", "--logdir=/logs",
- String.format("--path_prefix=%s", routePath)
- ));
- container.setImagePullPolicy("IfNotPresent");
- container.addPortsItem(new
V1ContainerPort().containerPort(TensorboardUtils.DEFAULT_TENSORBOARD_PORT));
- container.addVolumeMountsItem(new
V1VolumeMount().mountPath("/logs").name("volume"));
- deploymentTemplatePodSpec.addContainersItem(container);
-
- V1Volume volume = new V1Volume().name("volume");
- volume.setPersistentVolumeClaim(
- new V1PersistentVolumeClaimVolumeSource().claimName(pvc)
- );
- deploymentTemplatePodSpec.addVolumesItem(volume);
-
- deploymentTemplateSpec.setSpec(deploymentTemplatePodSpec);
-
- deploymentSpec.setTemplate(deploymentTemplateSpec);
-
- deployment.setSpec(deploymentSpec);
-
- return deployment;
- }
-
- public static V1Service parseService(String svcName, String podName) {
- V1Service svc = new V1Service();
- svc.metadata(new V1ObjectMeta().name(svcName));
-
- V1ServiceSpec svcSpec = new V1ServiceSpec();
- svcSpec.setSelector(Collections.singletonMap("app", podName)); // bind to
pod
- svcSpec.addPortsItem(new V1ServicePort().protocol("TCP").targetPort(
- new
IntOrString(TensorboardUtils.DEFAULT_TENSORBOARD_PORT)).port(TensorboardUtils.SERVICE_PORT));
- svc.setSpec(svcSpec);
- return svc;
- }
-
- public static IngressRoute parseIngressRoute(String ingressName, String
namespace,
- String routePath, String
svcName) {
-
- IngressRoute ingressRoute = new IngressRoute();
- ingressRoute.setMetadata(
- new V1ObjectMeta().name(ingressName).namespace((namespace))
- );
-
- IngressRouteSpec ingressRouteSpec = new IngressRouteSpec();
- ingressRouteSpec.setEntryPoints(new
HashSet<>(Collections.singletonList("web")));
- SpecRoute specRoute = new SpecRoute();
- specRoute.setKind("Rule");
- specRoute.setMatch(String.format("PathPrefix(`%s`)", routePath));
-
- Map<String, Object> service = new HashMap<String, Object>() {{
- put("kind", "Service");
- put("name", svcName);
- put("port", TensorboardUtils.SERVICE_PORT);
- put("namespace", namespace);
- }};
-
- specRoute.setServices(new HashSet<Map<String, Object>>() {{
- add(service);
- }});
-
- ingressRouteSpec.setRoutes(new HashSet<SpecRoute>() {{
- add(specRoute);
- }});
-
- ingressRoute.setSpec(ingressRouteSpec);
-
- return ingressRoute;
- }
-
-}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParser.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParser.java
index 6621c67..6e5803d 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParser.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParser.java
@@ -81,5 +81,4 @@ public class VolumeSpecParser {
return pvc;
}
-
}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/TensorboardUtils.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/TensorboardUtils.java
deleted file mode 100644
index a7d8c9f..0000000
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/util/TensorboardUtils.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.submarine.server.submitter.k8s.util;
-
-public class TensorboardUtils {
- /*
- Prefix constants
- */
- public static final String PV_PREFIX = "tfboard-pv-";
- public static final String HOST_PREFIX = "/tmp/tfboard-logs/";
- public static final String STORAGE = "1Gi";
- public static final String PVC_PREFIX = "tfboard-pvc-";
- public static final String DEPLOY_PREFIX = "tfboard-";
- public static final String POD_PREFIX = "tfboard-";
- public static final String IMAGE_NAME = "tensorflow/tensorflow:1.11.0";
- public static final String SVC_PREFIX = "tfboard-svc-";
- public static final String INGRESS_PREFIX = "tfboard-ingressroute";
- public static final String PATH_PREFIX = "/tfboard-";
- public static final Integer DEFAULT_TENSORBOARD_PORT = 6006;
- public static final Integer SERVICE_PORT = 8080;
-}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
index 4add1e0..173f473 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
@@ -25,6 +25,7 @@ import java.net.URISyntaxException;
import io.kubernetes.client.ApiException;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.server.api.experiment.Experiment;
+import org.apache.submarine.server.api.experiment.TensorboardInfo;
import org.apache.submarine.server.api.spec.ExperimentSpec;
import org.junit.Assert;
import org.junit.Before;
@@ -35,18 +36,17 @@ import org.slf4j.LoggerFactory;
/**
* We have two ways to test submitter for K8s cluster, local and travis CI.
* <p>
- * For running the tests locally, ensure that:
- * 1. There's a K8s cluster running somewhere
- * 2. Had set the env KUBECONFIG variable
- * 3. The CRDs was created in default namespace. The operator doesn't needs to
be running.
+ * For running the tests locally, ensure that: 1. There's a K8s cluster running
+ * somewhere 2. Had set the env KUBECONFIG variable 3. The CRDs was created in
+ * default namespace. The operator doesn't needs to be running.
* <p>
* Use "kubectl -n submarine get tfjob" or "kubectl -n submarine get
pytorchjob"
* to check the status if you comment the deletion job code in method "after()"
* <p>
* <p>
- * For the travis CI, we use the kind to setup K8s, more info see
'.travis.yml' file.
- * Local: docker run -it --privileged -p 8443:8443 -p 10080:10080
bsycorp/kind:latest-1.15
- * Travis: See '.travis.yml'
+ * For the travis CI, we use the kind to setup K8s, more info see '.travis.yml'
+ * file. Local: docker run -it --privileged -p 8443:8443 -p 10080:10080
+ * bsycorp/kind:latest-1.15 Travis: See '.travis.yml'
*/
public class K8SJobSubmitterTest extends SpecBuilder {
private static final Logger LOG =
LoggerFactory.getLogger(K8SJobSubmitterTest.class);
@@ -59,31 +59,35 @@ public class K8SJobSubmitterTest extends SpecBuilder {
}
@Test
- public void testRunPyTorchJobPerRequest() throws URISyntaxException,
- IOException, SubmarineRuntimeException {
+ public void testRunPyTorchJobPerRequest() throws URISyntaxException,
IOException,
+ SubmarineRuntimeException {
ExperimentSpec spec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, pytorchJobReqFile);
run(spec);
}
@Test
- public void testRunTFJobPerRequest() throws URISyntaxException,
- IOException, SubmarineRuntimeException {
+ public void testRunTFJobPerRequest() throws URISyntaxException, IOException,
SubmarineRuntimeException {
ExperimentSpec spec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, tfJobReqFile);
run(spec);
}
@Test
public void testCreateTFJob() throws IOException, URISyntaxException {
- ExperimentSpec spec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, tfTfboardJobwReqFile);
+ ExperimentSpec spec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, tfTfboardJobReqFile);
Experiment experiment = submitter.createExperiment(spec);
}
@Test
public void testDeleteTFJob() throws IOException, URISyntaxException {
- ExperimentSpec spec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, tfTfboardJobwReqFile);
+ ExperimentSpec spec = (ExperimentSpec)
buildFromJsonFile(ExperimentSpec.class, tfTfboardJobReqFile);
Experiment experiment = submitter.deleteExperiment(spec);
}
+ @Test
+ public void testGetTensorboardInfo() throws IOException, URISyntaxException {
+ TensorboardInfo tensorboardInfo = submitter.getTensorboardInfo();
+ }
+
private void run(ExperimentSpec spec) throws SubmarineRuntimeException {
// create
Experiment experimentCreated = submitter.createExperiment(spec);
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
index eb41aca..573c295 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/SpecBuilder.java
@@ -44,7 +44,7 @@ public abstract class SpecBuilder {
"/pytorch_job_req_http_git_code_localizer.json";
protected final String pytorchJobWithSSHGitCodeLocalizerFile =
"/pytorch_job_req_ssh_git_code_localizer.json";
- protected final String tfTfboardJobwReqFile = "/tf_tfboard_mnist_req.json";
+ protected final String tfTfboardJobReqFile = "/tf_tfboard_mnist_req.json";
protected Object buildFromJsonFile(Object obj, String filePath) throws
IOException,
URISyntaxException {
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/parser/TensorboardSpecParserTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/parser/TensorboardSpecParserTest.java
deleted file mode 100644
index 2503322..0000000
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/parser/TensorboardSpecParserTest.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.submarine.server.submitter.k8s.parser;
-
-import io.kubernetes.client.models.V1Deployment;
-import io.kubernetes.client.models.V1Service;
-import junit.framework.TestCase;
-import
org.apache.submarine.server.submitter.k8s.model.ingressroute.IngressRoute;
-import org.apache.submarine.server.submitter.k8s.util.TensorboardUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class TensorboardSpecParserTest extends TestCase {
-
- @Test
- public void testParseDeployment() {
- final String id = "123456789";
-
- final String name = TensorboardUtils.DEPLOY_PREFIX + id;
- final String image = TensorboardUtils.IMAGE_NAME;
- final String routePath = TensorboardUtils.PATH_PREFIX + id;
- final String pvc = TensorboardUtils.PVC_PREFIX + id;
-
- V1Deployment deployment = TensorboardSpecParser.parseDeployment(name,
image, routePath, pvc);
-
- Assert.assertEquals(name, deployment.getMetadata().getName());
- Assert.assertEquals(image,
-
deployment.getSpec().getTemplate().getSpec().getContainers().get(0).getImage());
-
- Assert.assertEquals(pvc,
- deployment.getSpec().getTemplate().getSpec().getVolumes()
- .get(0).getPersistentVolumeClaim().getClaimName());
- }
-
- @Test
- public void testParseService() {
- final String id = "123456789";
-
- final String svcName = TensorboardUtils.SVC_PREFIX + id;
- final String podName = TensorboardUtils.DEPLOY_PREFIX + id;
-
- V1Service svc = TensorboardSpecParser.parseService(svcName, podName);
-
- Assert.assertEquals(svcName, svc.getMetadata().getName());
- Assert.assertEquals(podName, svc.getSpec().getSelector().get("app"));
- }
-
- @Test
- public void testParseIngressRoute() {
- final String id = "123456789";
- final String namespace = "default";
-
- final String ingressName = TensorboardUtils.INGRESS_PREFIX + id;
- final String routePath = TensorboardUtils.PATH_PREFIX + id;
- final String svcName = TensorboardUtils.SVC_PREFIX + id;
-
- IngressRoute ingressRoute = TensorboardSpecParser.parseIngressRoute(
- ingressName, namespace, routePath, svcName
- );
-
- Assert.assertEquals(ingressRoute.getMetadata().getName(), ingressName);
- }
-}
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParserTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParserTest.java
deleted file mode 100644
index afacadb..0000000
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/parser/VolumeSpecParserTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.submarine.server.submitter.k8s.parser;
-import io.kubernetes.client.custom.Quantity;
-import io.kubernetes.client.models.V1PersistentVolume;
-import io.kubernetes.client.models.V1PersistentVolumeClaim;
-import org.apache.submarine.server.submitter.k8s.util.TensorboardUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class VolumeSpecParserTest {
- private static final Logger LOG =
LoggerFactory.getLogger(VolumeSpecParserTest.class);
-
- @Test
- public void testParsePersistentVolume() {
- final String id = "123456789";
-
- final String name = TensorboardUtils.PV_PREFIX + id;
- final String hostPath = TensorboardUtils.HOST_PREFIX + id;
- final String storage = TensorboardUtils.STORAGE;
-
- V1PersistentVolume pv = VolumeSpecParser.parsePersistentVolume(name,
hostPath, storage);
- LOG.info(pv.toString());
-
- Assert.assertEquals(name, pv.getMetadata().getName());
- Assert.assertEquals(hostPath, pv.getSpec().getHostPath().getPath());
- Assert.assertEquals(new Quantity(storage),
pv.getSpec().getCapacity().get("storage"));
- }
-
- @Test
- public void testParsePersistentVolumeClaim() {
- final String id = "123456789";
-
- final String name = TensorboardUtils.PVC_PREFIX + id;
- final String volume = TensorboardUtils.PV_PREFIX + id;
- final String storage = TensorboardUtils.STORAGE;
-
- V1PersistentVolumeClaim pvc =
VolumeSpecParser.parsePersistentVolumeClaim(name, volume, storage);
-
- LOG.info(pvc.toString());
- Assert.assertEquals(name, pvc.getMetadata().getName());
- Assert.assertEquals(volume, pvc.getSpec().getVolumeName());
- Assert.assertEquals(new Quantity(storage),
pvc.getSpec().getResources().getRequests().get("storage"));
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
![https://user-images.githubusercontent.com/24364830/104589865-7657f300-56a5-11eb-969d-60d7f4567625.jpg"](https://user-images.githubusercontent.com/24364830/104589865-7657f300-56a5-11eb-969d-60d7f4567625.jpg"){kind=link}