This is an automated email from the ASF dual-hosted git repository.
pingsutw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new f53e356 SUBMARINE-744. Frontend support for MLflow UI
f53e356 is described below
commit f53e3564a270e57690edf0e58a8137a78d9eca3e
Author: kobe860219 <[email protected]>
AuthorDate: Sun Mar 14 00:57:44 2021 +0800
SUBMARINE-744. Frontend support for MLflow UI
### What is this PR for?
Support for MLflow in workbench. Users click a button, and the submarine
will redirect them to MLflow UI.
### What type of PR is it?
[Feature]
### Todos
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-744
### How should this be tested?
https://travis-ci.org/github/kobe860219/submarine
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: kobe860219 <[email protected]>
Signed-off-by: Kevin <[email protected]>
Closes #536 from kobe860219/SUBMARINE-744 and squashes the following
commits:
d51d500c [kobe860219] SUBMARINE-744. Frontend support for MLflow UI
65a845b5 [kobe860219] SUBMARINE-744. Frontend support for MLflow UI
---
dev-support/docker-images/jupyter/Dockerfile | 2 +-
.../org/apache/submarine/server/api/Submitter.java | 9 +++++
.../server/api/experiment/MlflowInfo.java | 46 ++++++++++++++++++++++
.../server/experiment/ExperimentManager.java | 13 +++++-
.../submarine/server/rest/ExperimentRestApi.java | 19 +++++++++
.../server/submitter/k8s/K8sSubmitter.java | 40 +++++++++++++++++++
.../server/submitter/k8s/K8SJobSubmitterTest.java | 6 +++
.../src/app/interfaces/mlflow-info.ts | 23 +++++++++++
.../experiment-home/experiment-home.component.html | 11 ++++++
.../experiment-home/experiment-home.component.ts | 23 +++++++++++
.../src/app/services/experiment.service.ts | 14 +++++++
11 files changed, 204 insertions(+), 2 deletions(-)
diff --git a/dev-support/docker-images/jupyter/Dockerfile
b/dev-support/docker-images/jupyter/Dockerfile
index ad7aea3..a90070b 100644
--- a/dev-support/docker-images/jupyter/Dockerfile
+++ b/dev-support/docker-images/jupyter/Dockerfile
@@ -19,7 +19,7 @@ ARG NB_USER="jovyan"
ARG NB_UID="1000"
ARG NB_PREFIX="/"
ARG NB_PORT=8888
-ARG MLFLOW_TRACKING_URI="http://10.96.0.3:8080";
+ARG MLFLOW_TRACKING_URI="http://10.96.0.3:5000";
USER root
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
index c52450e..31f02a0 100644
---
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/Submitter.java
@@ -24,6 +24,7 @@ import
org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.ExperimentLog;
import org.apache.submarine.server.api.experiment.TensorboardInfo;
+import org.apache.submarine.server.api.experiment.MlflowInfo;
import org.apache.submarine.server.api.notebook.Notebook;
import org.apache.submarine.server.api.spec.ExperimentSpec;
import org.apache.submarine.server.api.spec.NotebookSpec;
@@ -128,4 +129,12 @@ public interface Submitter {
* @throws SubmarineRuntimeException running error
*/
TensorboardInfo getTensorboardInfo() throws SubmarineRuntimeException;
+
+ /**
+ * Get mlflow meta data
+ * @param
+ * @return object
+ * @throws SubmarineRuntimeException running error
+ */
+ MlflowInfo getMlflowInfo() throws SubmarineRuntimeException;
}
diff --git
a/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/MlflowInfo.java
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/MlflowInfo.java
new file mode 100644
index 0000000..3605a39
--- /dev/null
+++
b/submarine-server/server-api/src/main/java/org/apache/submarine/server/api/experiment/MlflowInfo.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.submarine.server.api.experiment;
+
+public class MlflowInfo {
+ public boolean available;
+ public String url;
+
+ public MlflowInfo(boolean available, String url) {
+ this.available = available;
+ this.url = url;
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ @Override
+ public String toString() {
+ return "MLflowInfo{" +
+ "available=" + available +
+ ", url='" + url + '\'' +
+ '}';
+ }
+}
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
index e8a6b97..ee429a3 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/experiment/ExperimentManager.java
@@ -39,6 +39,7 @@ import
org.apache.submarine.server.api.experiment.ExperimentId;
import org.apache.submarine.server.api.Submitter;
import org.apache.submarine.server.api.experiment.ExperimentLog;
import org.apache.submarine.server.api.experiment.TensorboardInfo;
+import org.apache.submarine.server.api.experiment.MlflowInfo;
import org.apache.submarine.server.api.spec.ExperimentSpec;
import org.apache.submarine.server.experiment.database.ExperimentEntity;
import org.apache.submarine.server.experiment.database.ExperimentService;
@@ -106,7 +107,7 @@ public class ExperimentManager {
spec.getMeta().getEnvVars().put(RestConstants.JOB_ID, id.toString());
spec.getMeta().getEnvVars().put(RestConstants.SUBMARINE_TRACKING_URI, url);
spec.getMeta().getEnvVars().put(RestConstants.LOG_DIR_KEY,
RestConstants.LOG_DIR_VALUE);
-
+
String lowerName = spec.getMeta().getName().toLowerCase();
spec.getMeta().setName(lowerName);
@@ -279,6 +280,16 @@ public class ExperimentManager {
return submitter.getTensorboardInfo();
}
+ /**
+ * Get mlflow meta data
+ *
+ * @return mlflowinfo
+ * @throws SubmarineRuntimeException the service error
+ */
+ public MlflowInfo getMLflowInfo() throws SubmarineRuntimeException {
+ return submitter.getMlflowInfo();
+ }
+
private void checkSpec(ExperimentSpec spec) throws SubmarineRuntimeException
{
if (spec == null) {
throw new SubmarineRuntimeException(Status.OK.getStatusCode(), "Invalid
experiment spec.");
diff --git
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java
index c3dc873..d914824 100644
---
a/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java
+++
b/submarine-server/server-core/src/main/java/org/apache/submarine/server/rest/ExperimentRestApi.java
@@ -42,6 +42,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.TensorboardInfo;
+import org.apache.submarine.server.api.experiment.MlflowInfo;
import org.apache.submarine.server.experiment.ExperimentManager;
import
org.apache.submarine.server.experimenttemplate.ExperimentTemplateManager;
import org.apache.submarine.server.api.experiment.ExperimentLog;
@@ -278,6 +279,24 @@ public class ExperimentRestApi {
}
}
+ @GET
+ @Path("/mlflow")
+ @Operation(summary = "Get mlflow's information",
+ tags = {"experiment"},
+ responses = {
+ @ApiResponse(description = "successful operation", content =
@Content(
+ schema = @Schema(implementation =
JsonResponse.class))),
+ @ApiResponse(responseCode = "404", description = "MLflow not
found")})
+ public Response getMLflowInfo() {
+ try {
+ MlflowInfo mlflowInfo = experimentManager.getMLflowInfo();
+ return new
JsonResponse.Builder<MlflowInfo>(Response.Status.OK).success(true)
+ .result(mlflowInfo).build();
+ } catch (SubmarineRuntimeException e) {
+ return parseExperimentServiceException(e);
+ }
+ }
+
private Response parseExperimentServiceException(SubmarineRuntimeException
e) {
return new JsonResponse.Builder<String>(e.getCode())
.message(e.getMessage().equals("Conflict") ? "Duplicated experiment
name" : e.getMessage()).build();
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
index 885e94c..5165173 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/main/java/org/apache/submarine/server/submitter/k8s/K8sSubmitter.java
@@ -54,6 +54,7 @@ import
org.apache.submarine.server.api.exception.InvalidSpecException;
import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.ExperimentLog;
import org.apache.submarine.server.api.experiment.TensorboardInfo;
+import org.apache.submarine.server.api.experiment.MlflowInfo;
import org.apache.submarine.server.api.notebook.Notebook;
import org.apache.submarine.server.api.spec.ExperimentMeta;
import org.apache.submarine.server.api.spec.ExperimentSpec;
@@ -300,6 +301,45 @@ public class K8sSubmitter implements Submitter {
}
@Override
+ public MlflowInfo getMlflowInfo() throws SubmarineRuntimeException {
+ final String name = "submarine-mlflow";
+ final String namespace = "default";
+ final String ingressRouteName = "submarine-mlflow-ingressroute";
+
+ try {
+ V1Deployment deploy = appsV1Api.readNamespacedDeploymentStatus(name,
namespace, "true");
+ boolean available = deploy.getStatus().getAvailableReplicas() > 0; // at
least one replica is running
+
+ IngressRoute ingressRoute = new IngressRoute();
+ V1ObjectMeta meta = new V1ObjectMeta();
+ meta.setName(ingressRouteName);
+ meta.setNamespace(namespace);
+ ingressRoute.setMetadata(meta);
+ Object object = api.getNamespacedCustomObject(
+ ingressRoute.getGroup(), ingressRoute.getVersion(),
+ ingressRoute.getMetadata().getNamespace(),
+ ingressRoute.getPlural(), ingressRouteName
+ );
+
+ Gson gson = new JSON().getGson();
+ String jsonString = gson.toJson(object);
+ IngressRoute result = gson.fromJson(jsonString, IngressRoute.class);
+
+
+ String route =
result.getSpec().getRoutes().stream().findFirst().get().getMatch();
+
+ String url = route.replace("PathPrefix(`", "").replace("`)", "/");
+
+ MlflowInfo mlflowInfo = new MlflowInfo(available, url);
+
+ return mlflowInfo;
+ } catch (ApiException e) {
+ throw new SubmarineRuntimeException(e.getCode(), e.getMessage());
+ }
+ }
+
+
+ @Override
public Notebook createNotebook(NotebookSpec spec) throws
SubmarineRuntimeException {
Notebook notebook;
final String name = spec.getMeta().getName();
diff --git
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
index 173f473..af7a841 100644
---
a/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
+++
b/submarine-server/server-submitter/submitter-k8s/src/test/java/org/apache/submarine/server/submitter/k8s/K8SJobSubmitterTest.java
@@ -26,6 +26,7 @@ import io.kubernetes.client.ApiException;
import org.apache.submarine.commons.utils.exception.SubmarineRuntimeException;
import org.apache.submarine.server.api.experiment.Experiment;
import org.apache.submarine.server.api.experiment.TensorboardInfo;
+import org.apache.submarine.server.api.experiment.MlflowInfo;
import org.apache.submarine.server.api.spec.ExperimentSpec;
import org.junit.Assert;
import org.junit.Before;
@@ -88,6 +89,11 @@ public class K8SJobSubmitterTest extends SpecBuilder {
TensorboardInfo tensorboardInfo = submitter.getTensorboardInfo();
}
+ @Test
+ public void testGetMlflowInfo() throws IOException, URISyntaxException {
+ MlflowInfo mlflowInfo = submitter.getMlflowInfo();
+ }
+
private void run(ExperimentSpec spec) throws SubmarineRuntimeException {
// create
Experiment experimentCreated = submitter.createExperiment(spec);
diff --git
a/submarine-workbench/workbench-web/src/app/interfaces/mlflow-info.ts
b/submarine-workbench/workbench-web/src/app/interfaces/mlflow-info.ts
new file mode 100644
index 0000000..649c431
--- /dev/null
+++ b/submarine-workbench/workbench-web/src/app/interfaces/mlflow-info.ts
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export interface MlflowInfo {
+ available: boolean;
+ url: string;
+}
diff --git
a/submarine-workbench/workbench-web/src/app/pages/workbench/experiment/experiment-home/experiment-home.component.html
b/submarine-workbench/workbench-web/src/app/pages/workbench/experiment/experiment-home/experiment-home.component.html
index 3fa8ffb..f2733b9 100644
---
a/submarine-workbench/workbench-web/src/app/pages/workbench/experiment/experiment-home/experiment-home.component.html
+++
b/submarine-workbench/workbench-web/src/app/pages/workbench/experiment/experiment-home/experiment-home.component.html
@@ -34,6 +34,17 @@
target="_blank"
nzType="primary"
style="margin: 0px 4px 0px 4px"
+ [nzLoading]="isMlflowLoading"
+ [href]="mlflowUrl"
+ >
+ <i nz-icon nzType="area-chart"></i>
+ MLflow UI
+ </a>
+ <a
+ nz-button
+ target="_blank"
+ nzType="primary"
+ style="margin: 0px 4px 0px 4px"
[nzLoading]="isTensorboardLoading"
[href]="tensorboardUrl"
>
diff --git
a/submarine-workbench/workbench-web/src/app/pages/workbench/experiment/experiment-home/experiment-home.component.ts
b/submarine-workbench/workbench-web/src/app/pages/workbench/experiment/experiment-home/experiment-home.component.ts
index f95304f..603e6f9 100644
---
a/submarine-workbench/workbench-web/src/app/pages/workbench/experiment/experiment-home/experiment-home.component.ts
+++
b/submarine-workbench/workbench-web/src/app/pages/workbench/experiment/experiment-home/experiment-home.component.ts
@@ -49,6 +49,10 @@ export class ExperimentHomeComponent implements OnInit {
reloadInterval = interval(this.reloadPeriod);
reloadSub = null;
+ //mlflow
+ isMlflowLoading: boolean = true;
+ mlflowUrl: string = '';
+
// tensorboard
isTensorboardLoading: boolean = true;
tensorboardUrl: string = '';
@@ -68,6 +72,7 @@ export class ExperimentHomeComponent implements OnInit {
this.experimentService.emitInfo(null);
this.getTensorboardInfo(1000, 50000);
+ this.getMlflowInfo(1000, 50000);
}
fetchExperimentList() {
@@ -164,4 +169,22 @@ export class ExperimentHomeComponent implements OnInit {
(err) => console.log(err)
);
}
+
+ getMlflowInfo(period: number, due: number) {
+ interval(period)
+ .pipe(
+ mergeMap(() => this.experimentService.getMlflowInfo()),
+ tap((x) => console.log(x)),
+ filter((res) => res.available),
+ take(1),
+ timeout(due)
+ )
+ .subscribe(
+ (res) => {
+ this.isMlflowLoading = !res.available;
+ this.mlflowUrl = res.url;
+ },
+ (err) => console.log(err)
+ );
+ }
}
diff --git
a/submarine-workbench/workbench-web/src/app/services/experiment.service.ts
b/submarine-workbench/workbench-web/src/app/services/experiment.service.ts
index 891b150..bd3eaae 100644
--- a/submarine-workbench/workbench-web/src/app/services/experiment.service.ts
+++ b/submarine-workbench/workbench-web/src/app/services/experiment.service.ts
@@ -25,6 +25,7 @@ import { ExperimentSpec } from
'@submarine/interfaces/experiment-spec';
import { ExperimentTemplate } from '@submarine/interfaces/experiment-template';
import { ExperimentTemplateSubmit } from
'@submarine/interfaces/experiment-template-submit';
import { TensorboardInfo } from '@submarine/interfaces/tensorboard-info';
+import { MlflowInfo } from '@submarine/interfaces/mlflow-info';
import { BaseApiService } from '@submarine/services/base-api.service';
import { of, throwError, Observable, Subject } from 'rxjs';
import { catchError, map, switchMap } from 'rxjs/operators';
@@ -225,6 +226,19 @@ export class ExperimentService {
);
}
+ getMlflowInfo(): Observable<MlflowInfo> {
+ const apiUrl = this.baseApi.getRestApi('/v1/experiment/mlflow');
+ return this.httpClient.get<Rest<MlflowInfo>>(apiUrl).pipe(
+ switchMap((res) => {
+ if (res.success) {
+ return of(res.result);
+ } else {
+ throw this.baseApi.createRequestError(res.message, res.code, apiUrl,
'get');
+ }
+ })
+ );
+ }
+
durationHandle(secs: number) {
const hr = Math.floor(secs / 3600);
const min = Math.floor((secs - hr * 3600) / 60);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
