This is an automated email from the ASF dual-hosted git repository.
wcjtw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 322d190 SUBMARINE-773. Support MLflow model registry (#542)
322d190 is described below
commit 322d190fb612089febbf909d55975749551dd921
Author: Wen-Chien, Juan <[email protected]>
AuthorDate: Sat May 1 16:23:04 2021 +0800
SUBMARINE-773. Support MLflow model registry (#542)
* SUBMARINE-773. Support MLflow model registry
* minio server done
* Model registry done
* SUBMARINE-773. Support MLflow model registry
* SUBMARINE-773. Support MLflow model registry
* Add version in jupyter notebook image
* Fix version
---
dev-support/docker-images/jupyter/Dockerfile | 7 ++
dev-support/docker-images/mlflow/Dockerfile | 19 +++-
dev-support/docker-images/mlflow/start.sh | 37 +++++++
.../submarine/templates/submarine-minio.yaml | 114 +++++++++++++++++++++
helm-charts/submarine/values.yaml | 3 +
5 files changed, 175 insertions(+), 5 deletions(-)
diff --git a/dev-support/docker-images/jupyter/Dockerfile
b/dev-support/docker-images/jupyter/Dockerfile
index a90070b..6815d1d 100644
--- a/dev-support/docker-images/jupyter/Dockerfile
+++ b/dev-support/docker-images/jupyter/Dockerfile
@@ -106,6 +106,9 @@ RUN cp
submarine/submarine-sdk/pysubmarine/example/submarine_experiment_sdk.ipyn
cp -r
submarine/submarine-sdk/pysubmarine/example/{data,deepfm_example.ipynb,deepfm.json}
$HOME && \
rm submarine -rf
+# Install mlflow & minio dependencies
+RUN pip install boto3==1.17.58 mlflow==1.15.0
+
# Install latest stable qlib
RUN pip install numpy==1.19.5 pyqlib==0.6.1
@@ -113,6 +116,10 @@ RUN pip install numpy==1.19.5 pyqlib==0.6.1
RUN wget
https://raw.githubusercontent.com/microsoft/qlib/main/examples/workflow_by_code.ipynb
-P $HOME
+ENV MLFLOW_S3_ENDPOINT_URL http://10.96.0.4:9000
+ENV AWS_ACCESS_KEY_ID submarine_minio
+ENV AWS_SECRET_ACCESS_KEY submarine_minio
+
EXPOSE $NB_PORT
ENTRYPOINT ["tini", "-g", "--"]
CMD ["start-notebook.sh"]
diff --git a/dev-support/docker-images/mlflow/Dockerfile
b/dev-support/docker-images/mlflow/Dockerfile
index e316cfc..b20b8b5 100644
--- a/dev-support/docker-images/mlflow/Dockerfile
+++ b/dev-support/docker-images/mlflow/Dockerfile
@@ -17,12 +17,21 @@ FROM python:3.7.0-slim
RUN pip install mlflow
-RUN mkdir /submarine-mlflow
+RUN apt-get update && apt-get -y install --no-install-recommends
default-libmysqlclient-dev libpq-dev build-essential sqlite3 wget
+
+RUN pip install mlflow==1.15.0 sqlalchemy==1.4.11 boto3==1.17.58
-WORKDIR /submarine-mlflow
+COPY start.sh /usr/local/bin
+
+WORKDIR /usr/local/bin
+
+RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x mc
+
+ENV MLFLOW_S3_ENDPOINT_URL http://10.96.0.4:9000
+ENV AWS_ACCESS_KEY_ID submarine_minio
+ENV AWS_SECRET_ACCESS_KEY submarine_minio
+ENV BACKEND_URI sqlite:///store.db
EXPOSE 5000
-CMD mlflow server \
- --host 0.0.0.0 \
- --static-prefix "/mlflow"
\ No newline at end of file
+CMD ./start.sh
\ No newline at end of file
diff --git a/dev-support/docker-images/mlflow/start.sh
b/dev-support/docker-images/mlflow/start.sh
new file mode 100755
index 0000000..26d43cb
--- /dev/null
+++ b/dev-support/docker-images/mlflow/start.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# description: Start and stop daemon script for.
+#
+
+set -euo pipefail
+
+MLFLOW_S3_ENDPOINT_URL="http://10.96.0.4:9000";
+AWS_ACCESS_KEY_ID="submarine_minio"
+AWS_SECRET_ACCESS_KEY="submarine_minio"
+BACKEND_URI="sqlite:///store.db"
+DEFAULT_ARTIFACT_ROOT="s3://mlflow"
+STATIC_PREFIX="/mlflow"
+
+/bin/bash -c "sqlite3 store.db"
+
+/bin/bash -c "sleep 60; ./mc config host add minio ${MLFLOW_S3_ENDPOINT_URL}
${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY}"
+
+/bin/bash -c "./mc mb minio/mlflow"
+
+/bin/bash -c "mlflow server --host 0.0.0.0 --backend-store-uri ${BACKEND_URI}
--default-artifact-root ${DEFAULT_ARTIFACT_ROOT} --static-prefix
${STATIC_PREFIX}"
\ No newline at end of file
diff --git a/helm-charts/submarine/templates/submarine-minio.yaml
b/helm-charts/submarine/templates/submarine-minio.yaml
new file mode 100644
index 0000000..8490919
--- /dev/null
+++ b/helm-charts/submarine/templates/submarine-minio.yaml
@@ -0,0 +1,114 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+ name: submarine-minio-pv
+spec:
+ accessModes:
+ - ReadWriteMany
+ capacity:
+ storage: "{{ .Values.submarine.minio.storage }}"
+{{- with .Values.submarine.storage }}
+ {{- if eq (.type | lower) "nfs" }}
+ nfs:
+ server: {{ .nfs.ip }}
+ path: {{ .nfs.path }}
+ {{- else }}
+ hostPath:
+ path: "{{ .host.path }}"
+ type: DirectoryOrCreate
+ {{- end }}
+{{- end}}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: submarine-minio-pvc
+spec:
+ accessModes:
+ - ReadWriteMany
+ storageClassName: ""
+ resources:
+ requests:
+ storage: "{{ .Values.submarine.minio.storage }}"
+ volumeName: submarine-minio-pv
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: submarine-minio
+spec:
+ selector:
+ matchLabels:
+ app: submarine-minio-pod
+ template:
+ metadata:
+ labels:
+ app: submarine-minio-pod
+ spec:
+ containers:
+ - name: submarine-minio-container
+ image: minio/minio:latest
+ imagePullPolicy: IfNotPresent
+ args:
+ - server
+ - /data
+ env:
+ - name: MINIO_ACCESS_KEY
+ value: "submarine_minio"
+ - name: MINIO_SECRET_KEY
+ value: "submarine_minio"
+ ports:
+ - containerPort: 9000
+ volumeMounts:
+ - mountPath: "/data"
+ name: "volume"
+ subPath: "submarine-minio"
+ volumes:
+ - name: "volume"
+ persistentVolumeClaim:
+ claimName: "submarine-minio-pvc"
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: submarine-minio-service
+spec:
+ clusterIP: 10.96.0.4
+ selector:
+ app: submarine-minio-pod
+ ports:
+ - protocol: TCP
+ port: 9000
+ targetPort: 9000
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+ name: submarine-minio-ingressroute
+spec:
+ entryPoints:
+ - web
+ routes:
+ - kind: Rule
+ match: "PathPrefix(`{{ .Values.submarine.minio.ingressPath }}`)"
+ services:
+ - kind: Service
+ name: submarine-minio-service
+ port: 9000
diff --git a/helm-charts/submarine/values.yaml
b/helm-charts/submarine/values.yaml
index 763a3fb..5d9952b 100644
--- a/helm-charts/submarine/values.yaml
+++ b/helm-charts/submarine/values.yaml
@@ -39,6 +39,9 @@ submarine:
mlflow:
storage: 10Gi
ingressPath: "/mlflow"
+ minio:
+ storage: 10Gi
+ ingressPath: "/minio"
storage:
type: host # "host" or "nfs"
host:
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
