This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new 0b26558 SUBMARINE-314. Creating a k8s environment locally using kind
0b26558 is described below
commit 0b26558e88e9730f725e278d11b86bbd5ca11856
Author: Xun Liu <[email protected]>
AuthorDate: Mon Dec 9 14:34:07 2019 +0800
SUBMARINE-314. Creating a k8s environment locally using kind
### What is this PR for?
Because the submarine on cloud is based on the k8s operating environment,
in order to be able to develop and test locally, we use kind to deploy a
complete k8s environment locally.
1. install k8s cluster
2. install docker registry
### What type of PR is it?
Feature
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/SUBMARINE-314
### How should this be tested?
* [CI Pass](https://travis-ci.org/liuxunorg/submarine/builds/622469321)
### Screenshots (if appropriate)
```
kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-576cbf47c7-mk5bb 0/1 Pending 0 27s
kube-system coredns-576cbf47c7-zbhgz 0/1 Pending 0 27s
kube-system ip-masq-agent-fdzr5 1/1 Running 0 27s
kube-system ip-masq-agent-w2pgc 1/1 Running 0 25s
kube-system kindnet-hbclv 1/1 Running 0 25s
kube-system kindnet-vlx47 1/1 Running 0 27s
kube-system kube-proxy-84c9z 1/1 Running 0 25s
kube-system kube-proxy-mvvbz 1/1 Running 0 27s
```
```
docker tag busybox localhost:5000/busybox
docker push localhost:5000/busybox
The push refers to repository [localhost:5000/busybox]
1da8e4c8d307: Pushed
latest: digest:
sha256:679b1c1058c1f2dc59a3ee70eed986a88811c0205c8ceea57cec5f22d2c3fbb1 size:
527
```
### Questions:
* Does the licenses files need update? Yes/No
* Is there breaking changes for older versions? Yes/No
* Does this needs documentation? Yes/No
Author: Xun Liu <[email protected]>
Closes #123 from liuxunorg/SUBMARINE-314 and squashes the following commits:
018aa2f [Xun Liu] Add auto download kubectl & kind desc.
8245746 [Xun Liu] SUBMARINE-314. Creating a k8s environment locally using
kind
---
.gitignore | 3 +
submarine-cloud/hack/kind-cluster-build.sh | 240 +++++++++++++++++++++++++++++
submarine-cloud/hack/lib.sh | 78 ++++++++++
3 files changed, 321 insertions(+)
diff --git a/.gitignore b/.gitignore
index f614fea..d957c81 100644
--- a/.gitignore
+++ b/.gitignore
@@ -79,3 +79,6 @@ derby.log
submarine-workbench/interpreter/spark-interpreter/metastore_db/
spark-1.*-bin-hadoop*
.spark-dist
+
+# submarine-cloud
+submarine-cloud/output/*
diff --git a/submarine-cloud/hack/kind-cluster-build.sh
b/submarine-cloud/hack/kind-cluster-build.sh
new file mode 100755
index 0000000..67a557a
--- /dev/null
+++ b/submarine-cloud/hack/kind-cluster-build.sh
@@ -0,0 +1,240 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
+cd $ROOT
+
+source $ROOT/hack/lib.sh
+
+hack::ensure_kubectl
+hack::ensure_kind
+
+usage() {
+ cat <<EOF
+This script use kind to create Kubernetes cluster, about kind please refer:
https://kind.sigs.k8s.io/
+* This script will automatically install kubectr-${KUBECTL_VERSION} and
kind-${KIND_VERSION} in ${OUTPUT_BIN}
+
+Options:
+ -h,--help prints the usage message
+ -n,--name name of the Kubernetes cluster,default value:
kind
+ -c,--nodeNum the count of the cluster nodes,default value: 1
+ -k,--k8sVersion version of the Kubernetes cluster,default
value: v1.12.8
+ -v,--volumeNum the volumes number of each kubernetes
node,default value: 1
+Usage:
+ $0 --name testCluster --nodeNum 4 --k8sVersion v1.12.9
+EOF
+}
+
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+ -n|--name)
+ clusterName="$2"
+ shift
+ shift
+ ;;
+ -c|--nodeNum)
+ nodeNum="$2"
+ shift
+ shift
+ ;;
+ -k|--k8sVersion)
+ k8sVersion="$2"
+ shift
+ shift
+ ;;
+ -v|--volumeNum)
+ volumeNum="$2"
+ shift
+ shift
+ ;;
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ *)
+ echo "unknown option: $key"
+ usage
+ exit 1
+ ;;
+esac
+done
+
+clusterName=${clusterName:-kind}
+nodeNum=${nodeNum:-1}
+k8sVersion=${k8sVersion:-v1.12.8}
+volumeNum=${volumeNum:-1}
+
+echo "clusterName: ${clusterName}"
+echo "nodeNum: ${nodeNum}"
+echo "k8sVersion: ${k8sVersion}"
+echo "volumeNum: ${volumeNum}"
+
+# check requirements
+for requirement in kind docker
+do
+ echo "############ check ${requirement} ##############"
+ if hash ${requirement} 2>/dev/null;then
+ echo "${requirement} have installed"
+ else
+ echo "this script needs ${requirement}, please install ${requirement}
first."
+ exit 1
+ fi
+done
+
+echo "############# start create cluster:[${clusterName}] #############"
+workDir=${HOME}/kind/${clusterName}
+mkdir -p ${workDir}
+
+data_dir=${workDir}/data
+
+echo "clean data dir: ${data_dir}"
+if [ -d ${data_dir} ]; then
+ rm -rf ${data_dir}
+fi
+
+configFile=${workDir}/kind-config.yaml
+
+cat <<EOF > ${configFile}
+kind: Cluster
+apiVersion: kind.sigs.k8s.io/v1alpha3
+nodes:
+- role: control-plane
+ extraPortMappings:
+ - containerPort: 5000
+ hostPort: 5000
+ listenAddress: 127.0.0.1
+ protocol: TCP
+EOF
+
+for ((i=0;i<${nodeNum};i++))
+do
+ mkdir -p ${data_dir}/worker${i}
+ cat <<EOF >> ${configFile}
+- role: worker
+ extraMounts:
+EOF
+ for ((k=1;k<=${volumeNum};k++))
+ do
+ mkdir -p ${data_dir}/worker${i}/vol${k}
+ cat <<EOF >> ${configFile}
+ - containerPath: /mnt/disks/vol${k}
+ hostPath: ${data_dir}/worker${i}/vol${k}
+EOF
+ done
+done
+
+echo "start to create k8s cluster"
+kind create cluster --config ${configFile} --image kindest/node:${k8sVersion}
--name=${clusterName}
+export KUBECONFIG="$(kind get kubeconfig-path --name=${clusterName})"
+
+echo "deploy docker registry in kind"
+registryNode=${clusterName}-control-plane
+registryNodeIP=$($KUBECTL_BIN get nodes ${registryNode} -o template
--template='{{range.status.addresses}}{{if eq .type
"InternalIP"}}{{.address}}{{end}}{{end}}')
+registryFile=${workDir}/registry.yaml
+
+cat <<EOF >${registryFile}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: registry
+spec:
+ selector:
+ matchLabels:
+ app: registry
+ template:
+ metadata:
+ labels:
+ app: registry
+ spec:
+ hostNetwork: true
+ nodeSelector:
+ kubernetes.io/hostname: ${registryNode}
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ operator: "Equal"
+ effect: "NoSchedule"
+ containers:
+ - name: registry
+ image: registry:2
+ volumeMounts:
+ - name: data
+ mountPath: /data
+ volumes:
+ - name: data
+ hostPath:
+ path: /data
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: registry-proxy
+ labels:
+ app: registry-proxy
+spec:
+ selector:
+ matchLabels:
+ app: registry-proxy
+ template:
+ metadata:
+ labels:
+ app: registry-proxy
+ spec:
+ hostNetwork: true
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: kubernetes.io/hostname
+ operator: NotIn
+ values:
+ - ${registryNode}
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ operator: "Equal"
+ effect: "NoSchedule"
+ containers:
+ - name: socat
+ image: alpine/socat:1.0.5
+ args:
+ - tcp-listen:5000,fork,reuseaddr
+ - tcp-connect:${registryNodeIP}:5000
+EOF
+$KUBECTL_BIN apply -f ${registryFile}
+
+echo "init submarine env"
+$KUBECTL_BIN create ns submarine-e2e
+
+echo "############# success create cluster:[${clusterName}] #############"
+
+echo "To start using your cluster, run:"
+echo " export KUBECONFIG=$(kind get kubeconfig-path --name=${clusterName})"
+echo ""
+echo <<EOF
+NOTE: In kind, nodes run docker network and cannot access host network.
+If you configured local HTTP proxy in your docker, images may cannot be pulled
+because http proxy is inaccessible.
+
+If you cannot remove http proxy settings, you can either whitelist image
+domains in NO_PROXY environment or use 'docker pull <image> && kind load
+docker-image <image>' command to load images into nodes.
+EOF
diff --git a/submarine-cloud/hack/lib.sh b/submarine-cloud/hack/lib.sh
new file mode 100755
index 0000000..f3383e1
--- /dev/null
+++ b/submarine-cloud/hack/lib.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if [ -z "$ROOT" ]; then
+ echo "error: ROOT should be initialized"
+ exit 1
+fi
+
+OS=$(go env GOOS)
+ARCH=$(go env GOARCH)
+OUTPUT=${ROOT}/output
+OUTPUT_BIN=${OUTPUT}/bin
+KUBECTL_VERSION=1.12.10
+KUBECTL_BIN=$OUTPUT_BIN/kubectl
+KIND_VERSION=0.6.0
+KIND_BIN=$OUTPUT_BIN/kind
+
+test -d "$OUTPUT_BIN" || mkdir -p "$OUTPUT_BIN"
+
+function hack::verify_kubectl() {
+ if test -x "$KUBECTL_BIN"; then
+ [[ "$($KUBECTL_BIN version --client --short | grep -o -E
'[0-9]+\.[0-9]+\.[0-9]+')" == "$KUBECTL_VERSION" ]]
+ return
+ fi
+ return 1
+}
+
+function hack::ensure_kubectl() {
+ if hack::verify_kubectl; then
+ return 0
+ fi
+ echo "Installing kubectl v$KUBECTL_VERSION..."
+ tmpfile=$(mktemp)
+ trap "test -f $tmpfile && rm $tmpfile" RETURN
+ curl --retry 10 -L -o $tmpfile
https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/${OS}/${ARCH}/kubectl
+ mv $tmpfile $KUBECTL_BIN
+ chmod +x $KUBECTL_BIN
+}
+
+function hack::verify_kind() {
+ if test -x "$KIND_BIN"; then
+ [[ "$($KIND_BIN --version 2>&1 | cut -d ' ' -f 3)" == "$KIND_VERSION"
]]
+ return
+ fi
+ return 1
+}
+
+function hack::ensure_kind() {
+ if hack::verify_kind; then
+ return 0
+ fi
+ echo "Installing kind v$KIND_VERSION..."
+ tmpfile=$(mktemp)
+ trap "test -f $tmpfile && rm $tmpfile" RETURN
+ curl --retry 10 -L -o $tmpfile
https://github.com/kubernetes-sigs/kind/releases/download/v${KIND_VERSION}/kind-$(uname)-amd64
+ mv $tmpfile $KIND_BIN
+ chmod +x $KIND_BIN
+}
+
+# hack::version_ge "$v1" "$v2" checks whether "v1" is greater or equal to "v2"
+function hack::version_ge() {
+ [ "$(printf '%s\n' "$1" "$2" | sort -V | head -n1)" = "$2" ]
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
