This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 7ae1592 SUBMARINE-367. Support submarine configuration outside k8s
7ae1592 is described below
commit 7ae15921fd8d329abcba21c91ae473c023900213
Author: Xun Liu <liu...@apache.org>
AuthorDate: Fri Jan 31 21:27:08 2020 +0800
SUBMARINE-367. Support submarine configuration outside k8s
### What is this PR for?
We need to enable the submarine running in k8s to use an external
submarine-site.xml configuration file.
In this way, the user can configure the submarine-site.xml file externally,
and then allow the submarine running in k8s to run according to the user's
settings.
#### Test
1. Install k8s use by KIND
```
./submarine-cloud/hack/kind-cluster-build.sh
```
2. Install submarine cluster on k8s
```
./submarine-cloud/hack/deploy-submarine.sh
```
modify ./submarine-cloud/hack/conf/submarine-site.xml & log4j.properties
submarine-site.xml & log4j.properties automation mount to submarine-server
pod
You can input `http://127.0.0.1` to chrome open submarine workbench web.
3. uninstall submarine cluster on k8s
```
./submarine-cloud/hack/deploy-submarine.sh -u
```
### What type of PR is it?
[Feature]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/SUBMARINE-367
### How should this be tested?
* https://travis-ci.org/liuxunorg/submarine/builds/644276787
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Xun Liu <liu...@apache.org>
Closes #170 from liuxunorg/SUBMARINE-367 and squashes the following commits:
d52201b [Xun Liu] Fixed code style
183784a [Xun Liu] SUBMARINE-367. Make submarine configuration outside k8s
---
.gitignore | 1 +
submarine-cloud/hack/deploy-submarine.sh | 131 +++++++++++++++++++++
submarine-cloud/hack/kind-cluster-build.sh | 41 +------
submarine-cloud/hack/lib.sh | 22 ++++
.../{service.yaml => submarine-database.yaml} | 50 ++++----
.../{deployment.yaml => submarine-server.yaml} | 69 ++++++-----
6 files changed, 222 insertions(+), 92 deletions(-)
diff --git a/.gitignore b/.gitignore
index 195cfd6..3c4db43 100644
--- a/.gitignore
+++ b/.gitignore
@@ -83,3 +83,4 @@ spark-1.*-bin-hadoop*
# submarine-cloud
submarine-cloud/vendor/*
submarine-cloud/output/*
+submarine-cloud/hack/conf/*
diff --git a/submarine-cloud/hack/deploy-submarine.sh
b/submarine-cloud/hack/deploy-submarine.sh
new file mode 100755
index 0000000..52ece6d
--- /dev/null
+++ b/submarine-cloud/hack/deploy-submarine.sh
@@ -0,0 +1,131 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
+cd $ROOT
+SUBMARINE_HOME=${ROOT}/..
+
+source $ROOT/hack/lib.sh
+
+# Check requirements
+hack::check_requirements
+
+# Install submarine in k8s cluster
+function install_submarine() {
+ if [ ! -d "${ROOT}/hack/conf" ]; then
+ mkdir "${ROOT}/hack/conf"
+ fi
+
+ if [ ! -f "${ROOT}/hack/conf/submarine-site.xml" ]; then
+ cp "${SUBMARINE_HOME}/conf/submarine-site.xml.template"
"${ROOT}/hack/conf/submarine-site.xml"
+
+ # Replace the mysql jdbc.url in the submarine-site.xml file with the name
of the submarine database ip/service
+ sed -i.bak "s/127.0.0.1:3306/${DATABASE_IP}:3306/g"
"${ROOT}/hack/conf/submarine-site.xml"
+ fi
+
+ if [ ! -f "${ROOT}/hack/conf/log4j.properties" ]; then
+ cp ${SUBMARINE_HOME}/conf/log4j.properties.template
${ROOT}/hack/conf/log4j.properties
+ fi
+
+ echo ""
+ echo -e "Have you configured the
\033[31m${ROOT}/hack/conf/submarine-site.xml\033[0m file?"
+ echo -e "Have you configured the
\033[31m${ROOT}/hack/conf/log4j.properties\033[0m file?"
+ echo -n "Do you want to deploy submarine in k8s cluster now? [y/n]"
+ read myselect
+ if [[ "$myselect" == "y" || "$myselect" == "Y" ]]; then
+ if kubectl get configmap --namespace default | grep submarine-config
>/dev/null ; then
+ kubectl delete configmap --namespace default submarine-config
+ fi
+ kubectl create configmap --namespace default submarine-config
--from-file=${ROOT}/hack/conf/submarine-site.xml
--from-file=${ROOT}/hack/conf/log4j.properties
+
+ docker pull apache/submarine:operator-0.3.0-SNAPSHOT
+ kind load docker-image apache/submarine:operator-0.3.0-SNAPSHOT
+ kubectl apply -f $ROOT/manifests/submarine-operator/
+
+ docker pull apache/submarine:database-0.3.0-SNAPSHOT
+ kind load docker-image apache/submarine:database-0.3.0-SNAPSHOT
+ docker pull apache/submarine:server-0.3.0-SNAPSHOT
+ kind load docker-image apache/submarine:server-0.3.0-SNAPSHOT
+ kubectl apply -f $ROOT/manifests/submarine-cluster/
+
+ cat <<EOF
+NOTE: You can open your browser and access the submarine workbench at
http://127.0.0.1/
+EOF
+ fi
+}
+
+# Uninstall submarine in k8s cluster
+function uninstall_submarine() {
+ if kubectl get configmap --namespace default | grep submarine-config
>/dev/null ; then
+ kubectl delete configmap --namespace default submarine-config
+ fi
+ kubectl delete -f $ROOT/manifests/submarine-operator/
+ kubectl delete -f $ROOT/manifests/submarine-cluster/
+
+ cat <<EOF
+NOTE: Submarine cluster has been deleted
+EOF
+}
+
+usage() {
+ cat <<EOF
+This script use kind to create Submarine cluster, about kind please refer:
https://kind.sigs.k8s.io/
+* This script will automatically install kubectr-${KUBECTL_VERSION} and
kind-${KIND_VERSION} in ${OUTPUT_BIN}
+
+Options:
+ -d,--database ip/service of submarine database, default
value: submarine-database
+ -u,--uninstall uninstall submarine cluster
+ -h,--help prints the usage message
+Usage:
+ install: $0 --database database_ip
+ OR
+ unstall: $0 -u
+EOF
+}
+
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+ -d|--database)
+ DATABASE_IP="$2"
+ shift
+ shift
+ ;;
+ -u|--uninstall)
+ UNINSTALL="TRUE"
+ shift
+ ;;
+ *)
+ echo "unknown option: $key"
+ usage
+ exit 1
+ ;;
+esac
+done
+
+DATABASE_IP=${DATABASE_IP:-submarine-database}
+echo "Submarine database ip: ${DATABASE_IP}"
+
+if [[ "$UNINSTALL" == "TRUE" ]]; then
+ uninstall_submarine
+else
+ install_submarine
+fi
diff --git a/submarine-cloud/hack/kind-cluster-build.sh
b/submarine-cloud/hack/kind-cluster-build.sh
index d799eed..be0ceda 100755
--- a/submarine-cloud/hack/kind-cluster-build.sh
+++ b/submarine-cloud/hack/kind-cluster-build.sh
@@ -85,25 +85,8 @@ echo "nodeNum: ${nodeNum}"
echo "k8sVersion: ${k8sVersion}"
echo "volumeNum: ${volumeNum}"
-# check requirements
-for requirement in kind docker kubectl
-do
- echo "############ check ${requirement} ##############"
- if hash ${requirement} 2>/dev/null;then
- echo "${requirement} have installed"
- else
- echo "this script needs ${requirement}, please install ${requirement}
first."
- if test ${requirement} = "kind"; then
- hack::ensure_kind
- echo "Please add $KIND_BIN to PATH variable or copy it to one of
the locations $PATH"
- fi
- if test ${requirement} = "kubectl"; then
- hack::ensure_kubectl
- echo "Please add $KUBECTL_BIN to PATH variable or copy it to one
of the locations $PATH"
- fi
- exit 1
- fi
-done
+# Check requirements
+hack::check_requirements
echo "############# start create cluster:[${clusterName}] #############"
workDir=${HOME}/kind/${clusterName}
@@ -267,23 +250,3 @@ If you cannot remove http proxy settings, you can either
whitelist image
domains in NO_PROXY environment or use 'docker pull <image> && kind load
docker-image <image>' command to load images into nodes.
EOF
-
-# Run submarine in kind cluster
-echo ""
-echo -n "Do you want to run submarine in kind cluster now? [y/n]"
-read myselect
-if [[ "$myselect" == "y" || "$myselect" == "Y" ]]; then
- docker pull apache/submarine:operator-0.3.0-SNAPSHOT
- kind load docker-image apache/submarine:operator-0.3.0-SNAPSHOT
- kubectl apply -f $ROOT/manifests/submarine-operator/
-
- docker pull apache/submarine:database-0.3.0-SNAPSHOT
- kind load docker-image apache/submarine:database-0.3.0-SNAPSHOT
- docker pull apache/submarine:server-0.3.0-SNAPSHOT
- kind load docker-image apache/submarine:server-0.3.0-SNAPSHOT
- kubectl apply -f $ROOT/manifests/submarine-cluster/
-
- cat <<EOF
-NOTE: You can open your browser and access the submarine workbench at
http://127.0.0.1/
-EOF
-fi
diff --git a/submarine-cloud/hack/lib.sh b/submarine-cloud/hack/lib.sh
index f3383e1..782fd1c 100755
--- a/submarine-cloud/hack/lib.sh
+++ b/submarine-cloud/hack/lib.sh
@@ -76,3 +76,25 @@ function hack::ensure_kind() {
function hack::version_ge() {
[ "$(printf '%s\n' "$1" "$2" | sort -V | head -n1)" = "$2" ]
}
+
+function hack::check_requirements() {
+ # Check requirements
+ for requirement in kind docker kubectl
+ do
+ echo "############ check ${requirement} ##############"
+ if hash ${requirement} 2>/dev/null;then
+ echo "${requirement} have installed"
+ else
+ echo "this script needs ${requirement}, please install
${requirement} first."
+ if test ${requirement} = "kind"; then
+ hack::ensure_kind
+ echo "Please add $KIND_BIN to PATH variable or copy it to one
of the locations $PATH"
+ fi
+ if test ${requirement} = "kubectl"; then
+ hack::ensure_kubectl
+ echo "Please add $KUBECTL_BIN to PATH variable or copy it to
one of the locations $PATH"
+ fi
+ exit 1
+ fi
+ done
+}
diff --git a/submarine-cloud/manifests/submarine-cluster/service.yaml
b/submarine-cloud/manifests/submarine-cluster/submarine-database.yaml
similarity index 57%
rename from submarine-cloud/manifests/submarine-cluster/service.yaml
rename to submarine-cloud/manifests/submarine-cluster/submarine-database.yaml
index 7697101..070338c 100644
--- a/submarine-cloud/manifests/submarine-cluster/service.yaml
+++ b/submarine-cloud/manifests/submarine-cluster/submarine-database.yaml
@@ -14,32 +14,38 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-kind: Service
-apiVersion: v1
+apiVersion: apps/v1beta1
+kind: Deployment
metadata:
- name: submarine-svc
+ name: submarine-database
spec:
+ replicas: 1
selector:
- app: cluster-test
- ports:
- - port: 8080
+ matchLabels:
+ app: submarine-database
+ template:
+ metadata:
+ labels:
+ app: submarine-database
+ spec:
+ containers:
+ - name: submarine-database
+ image: apache/submarine:database-0.3.0-SNAPSHOT
+ ports:
+ - containerPort: 3306
+ env:
+ - name: MYSQL_ROOT_PASSWORD
+ value: "password" # Same submarine-site.xml mysql password
---
-apiVersion: extensions/v1beta1
-kind: Ingress
+apiVersion: v1
+kind: Service
metadata:
- name: submarine-ingress
- annotations:
- ingress.kubernetes.io/rewrite-target: /
+ name: submarine-database
spec:
- rules:
- - http:
- paths:
- - path: /
- backend:
- serviceName: submarine-svc
- servicePort: 8080
-
----
-# You can also access the submarine workbench via port-forward
-# kubectl port-forward svc/submarine-svc 18080:8080 --address 0.0.0.0
+ ports:
+ - name: submarine-database
+ port: 3306
+ targetPort: 3306
+ selector:
+ app: submarine-database
diff --git a/submarine-cloud/manifests/submarine-cluster/deployment.yaml
b/submarine-cloud/manifests/submarine-cluster/submarine-server.yaml
similarity index 68%
rename from submarine-cloud/manifests/submarine-cluster/deployment.yaml
rename to submarine-cloud/manifests/submarine-cluster/submarine-server.yaml
index 4ac9099..b9a0819 100644
--- a/submarine-cloud/manifests/submarine-cluster/deployment.yaml
+++ b/submarine-cloud/manifests/submarine-cluster/submarine-server.yaml
@@ -29,9 +29,6 @@ spec:
app: cluster-test
spec:
ServiceAccountName: "submarine-node"
- volumes:
- - name: data
- emptyDir: {}
containers:
- name: submarine-node
image: "apache/submarine:server-0.3.0-SNAPSHOT"
@@ -68,40 +65,50 @@ spec:
periodSeconds: 10
successThreshold: 1
failureThreshold: 3
+ volumeMounts:
+ - name: submarine-configmap
+ mountPath: /opt/submarine-current/conf/submarine-site.xml
+ subPath: submarine-site.xml
+ - name: submarine-configmap
+ mountPath: /opt/submarine-current/conf/log4j.properties
+ subPath: log4j.properties
+ volumes:
+ - name: submarine-configmap
+ configMap:
+ name: submarine-config
+ items:
+ - key: submarine-site.xml
+ path: submarine-site.xml
+ - key: log4j.properties
+ path: log4j.properties
---
-apiVersion: apps/v1beta1
-kind: Deployment
+kind: Service
+apiVersion: v1
metadata:
- name: submarine-database
+ name: submarine-svc
spec:
- replicas: 1
selector:
- matchLabels:
- app: submarine-database
- template:
- metadata:
- labels:
- app: submarine-database
- spec:
- containers:
- - name: submarine-database
- image: apache/submarine:database-0.3.0-SNAPSHOT
- ports:
- - containerPort: 3306
- env:
- - name: MYSQL_ROOT_PASSWORD
- value: "password" # Same submarine-site.xml mysql password
+ app: cluster-test
+ ports:
+ - port: 8080
---
-apiVersion: v1
-kind: Service
+apiVersion: extensions/v1beta1
+kind: Ingress
metadata:
- name: submarine-database
+ name: submarine-ingress
+ annotations:
+ ingress.kubernetes.io/rewrite-target: /
spec:
- ports:
- - name: submarine-database
- port: 3306
- targetPort: 3306
- selector:
- app: submarine-database
+ rules:
+ - http:
+ paths:
+ - path: /
+ backend:
+ serviceName: submarine-svc
+ servicePort: 8080
+
+---
+# You can also access the submarine workbench via port-forward
+# kubectl port-forward svc/submarine-svc 18080:8080 --address 0.0.0.0
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@submarine.apache.org
For additional commands, e-mail: dev-h...@submarine.apache.org
