Revert "YARN-8870. [Submarine] Add submarine installation scripts. (Xun Liu via wangda)"
This reverts commit 46d6e0016610ced51a76189daeb3ad0e3dbbf94c. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/228156cf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/228156cf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/228156cf Branch: refs/heads/HDFS-12943 Commit: 228156cfd1b474988bc4fedfbf7edddc87db41e3 Parents: 9287ab3 Author: Wangda Tan <wan...@apache.org> Authored: Tue Dec 4 14:13:06 2018 -0800 Committer: Wangda Tan <wan...@apache.org> Committed: Tue Dec 4 14:13:06 2018 -0800 ---------------------------------------------------------------------- .../resources/assemblies/hadoop-yarn-dist.xml | 8 - .../installation/install.conf | 74 ---- .../installation/install.sh | 116 ----- .../package/calico/calico-node.service | 50 --- .../installation/package/calico/calicoctl.cfg | 22 - .../installation/package/docker/daemon.json | 23 - .../installation/package/docker/docker.service | 35 -- .../installation/package/etcd/etcd.service | 40 -- .../package/hadoop/container-executor.cfg | 41 -- .../installation/package/submarine/submarine.sh | 25 -- .../installation/scripts/calico.sh | 224 ---------- .../installation/scripts/docker.sh | 166 ------- .../installation/scripts/download-server.sh | 42 -- .../installation/scripts/environment.sh | 213 --------- .../installation/scripts/etcd.sh | 152 ------- .../installation/scripts/hadoop.sh | 117 ----- .../installation/scripts/menu.sh | 444 ------------------- .../installation/scripts/nvidia-docker.sh | 99 ----- .../installation/scripts/nvidia.sh | 120 ----- .../installation/scripts/submarine.sh | 38 -- .../installation/scripts/utils.sh | 123 ----- 21 files changed, 2172 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 69ab66a..4055acb 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -66,14 +66,6 @@ <fileMode>0755</fileMode> </fileSet> <fileSet> - <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation</directory> - <outputDirectory>/share/hadoop/yarn/submarine-installer</outputDirectory> - <includes> - <include>**/*</include> - </includes> - <fileMode>0755</fileMode> - </fileSet> - <fileSet> <directory>hadoop-yarn/conf</directory> <outputDirectory>etc/hadoop</outputDirectory> <includes> http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf deleted file mode 100644 index 82dcf61..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.conf +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -##### install config ##### - -# DNS -LOCAL_DNS_HOST="172.17.0.9" # /etc/resolv.conf -YARN_DNS_HOST="10.196.69.173" # yarn dns server ip address - -# etcd hosts list -ETCD_HOSTS=(10.196.69.173 10.196.69.174 10.196.69.175) - -# docker registry ip:port -DOCKER_REGISTRY="10.120.196.232:5000" - -# Start the http download service on the specified server, -# Will download all the dependencies in the http server, -# Run the install script on other servers. -# Automatically download dependencies from http, -# Solve the problem that all servers are slow to download online. -# At the same time, you can also manually download the dependencies to the downloads directory. -# Offline installation of the system -DOWNLOAD_SERVER_IP="10.120.196.236" -DOWNLOAD_SERVER_PORT="19000" - -# yarn container-executor config -# How to compile container-executor: -# Go to the hadoop/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager -# path and enter the command: mvn package -Pnative -DskipTests -# Only the nodemanager is compiled. The path of the compiled container-executor is: -# ./target/native/target/usr/local/bin/container-executor -YARN_CONTAINER_EXECUTOR_PATH="Please enter the full path to the container-executor here" - -# Keep the same configuration as 'yarn.nodemanager.linux-container-executor.cgroups.hierarchy' -# in yarn-site.xml, default '/hadoop-yarn' -YARN_HIERARCHY="/hadoop-yarn" - -# Keep the same configuration as 'yarn.nodemanager.local-dirs' in yarn-site.xml -YARN_NODEMANAGER_LOCAL_DIRS="/home/hadoop/disk/1/yarn/local,/home/hadoop/disk/2/yarn/local,/home/hadoop/disk/3/yarn/local" - -# Keep the same configuration as 'yarn.nodemanager.log-dirs' in yarn-site.xml -YARN_NODEMANAGER_LOG_DIRS="/home/hadoop/disk/1/yarn/logs,/home/hadoop/disk/2/yarn/logs,/home/hadoop/disk/3/yarn/logs" - -##### System component download url address, Generally do not need to be modified ##### -DOCKER_REPO="https://yum.dockerproject.org/repo/main/centos/7/Packages"; -DOCKER_ENGINE_RPM="docker-engine-1.12.5-1.el7.centos.x86_64.rpm" -DOCKER_ENGINE_SELINUX_RPM="docker-engine-selinux-1.12.5-1.el7.centos.noarch.rpm" - -NVIDIA_DETECT_URL="https://raw.githubusercontent.com/liuxunorg/submarine-installer/packages/nvidia/nvidia-detect"; -NVIDIA_DOCKER_RPM="nvidia-docker-1.0.1-1.x86_64.rpm" -NVIDIA_DOCKER_RPM_URL="https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/${NVIDIA_DOCKER_RPM}"; - -ETCD_TAR_GZ="etcd-v3.3.9-linux-amd64.tar.gz" -ETCD_DOWNLOAD_URL="https://github.com/etcd-io/etcd/releases/download/v3.3.9/${ETCD_TAR_GZ}"; - -CALICO_IPV4POOL_CIDR="192.20.0.0" -CALICO_NETWORK_NAME="calico-network" -CALICO_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico"; -CALICO_IPAM_DOWNLOAD_URL="https://github.com/projectcalico/cni-plugin/releases/download/v1.11.7/calico-ipam"; -CALICOCTL_DOWNLOAD_URL="https://github.com/projectcalico/calicoctl/releases/download/v3.2.3/calicoctl"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh deleted file mode 100755 index 90b12c2..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/install.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# description: sumbarine install scripts. - -ROOT=$(cd "$(dirname "$0")"; pwd) -SUBMARINE_INSTALLER_VERSION="v0.7" -PACKAGE_DIR=${ROOT}/package -SCRIPTS_DIR=${ROOT}/scripts -INSTALL_TEMP_DIR=${ROOT}/temp -DOWNLOAD_DIR=${ROOT}/downloads -DATE=`date +%Y%m%d-%H:%M:%S` -LOG=${ROOT}/logs/install.log.`date +%Y%m%d%H%M%S` -LOCAL_HOST_IP_LIST=() -LOCAL_HOST_IP="" -OPERATING_SYSTEM="" -DOWNLOAD_HTTP="" - -# import shell script -. ${ROOT}/install.conf -. ${ROOT}/scripts/calico.sh -. ${ROOT}/scripts/docker.sh -. ${ROOT}/scripts/download-server.sh -. ${ROOT}/scripts/environment.sh -. ${ROOT}/scripts/etcd.sh -. ${ROOT}/scripts/hadoop.sh -. ${ROOT}/scripts/menu.sh -. ${ROOT}/scripts/nvidia.sh -. ${ROOT}/scripts/nvidia-docker.sh -. ${ROOT}/scripts/submarine.sh -. ${ROOT}/scripts/utils.sh - -#================================= Main ======================================== -mkdir $ROOT/logs/ -p -mkdir $INSTALL_TEMP_DIR -p -mkdir $DOWNLOAD_DIR -p - -source /etc/os-release -OPERATING_SYSTEM=$ID - -get_ip_list -ipCount=${#LOCAL_HOST_IP_LIST[@]} -if [[ $ipCount -eq 1 ]]; then - LOCAL_HOST_IP=${LOCAL_HOST_IP_LIST[0]} -else - echo -e "Detect the network card IP in the server, \e[31m[${LOCAL_HOST_IP_LIST[@]}]\e[0m" - echo -n -e "please enter a valid IP address: " - - read ipInput - if ! valid_ip $ipInput; then - echo -e "you input \e[31m$ipInput\e[0m address format is incorrect! " | tee -a $LOG - exit_install - else - LOCAL_HOST_IP=$ipInput - fi -fi - -echo -n -e "Please confirm whether the IP address of this machine is \e[31m${LOCAL_HOST_IP}\e[0m?[y|n]" -read myselect -if [[ "$myselect" != "y" && "$myselect" != "Y" ]]; then - exit_install -fi - -check_install_conf - -if [[ -n "$DOWNLOAD_SERVER_IP" && -n "$DOWNLOAD_SERVER_PORT" && "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then - DOWNLOAD_HTTP="http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT}"; -fi - -check_install_user - -# æ¸ çå®è£ 临æ¶ç®å½ -rm $INSTALL_TEMP_DIR/* -rf >>$LOG 2>&1 - -menu_index="0" -for ((j=1;;j++)) -do - menu - case "$menu_index" in - "0") - menu_index="$menu_choice" - ;; - "1"|"2"|"3"|"4"|"5") -# echo "aaaa=$menu_index-$menu_choice" - menu_process - if [[ $? = 1 ]]; then - echo "Press any key to return menu!" - read - fi - ;; - "a") - exit_install - ;; - "q") - exit_install - ;; - *) - menu_index="0" - menu_choice="0" - menu - ;; - esac -done http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service deleted file mode 100644 index 744c419..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calico-node.service +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[Unit] -Description=calico node -After=docker.service -Requires=docker.service - -[Service] -User=root -PermissionsStartOnly=true -ExecStart=/usr/bin/docker run --net=host --privileged --name=calico-node \ - -e ETCD_ENDPOINTS=ETCD_ENDPOINTS_REPLACE \ - -e CALICO_LIBNETWORK_ENABLED=true \ - -e CALICO_NETWORKING_BACKEND=bird \ - -e CALICO_DISABLE_FILE_LOGGING=true \ - -e CALICO_IPV4POOL_CIDR=CALICO_IPV4POOL_CIDR_REPLACE/16 \ - -e CALICO_IPV4POOL_IPIP=always \ - -e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \ - -e FELIX_IPV6SUPPORT=false \ - -e FELIX_LOGSEVERITYSCREEN=info \ - -e FELIX_IPINIPMTU=1440 \ - -e FELIX_HEALTHENABLED=true \ - -e IP= \ - -v /etc/calico/ssl:/etc/calico/ssl \ - -v /var/run/calico:/var/run/calico \ - -v /lib/modules:/lib/modules \ - -v /run/docker/plugins:/run/docker/plugins \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /var/log/calico:/var/log/calico \ - calico/node:v2.6.2 -ExecStop=/usr/bin/docker rm -f calico-node -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg deleted file mode 100644 index 44ada86..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/calico/calicoctl.cfg +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -apiVersion: v1 -kind: calicoApiConfig -metadata: -spec: - datastoreType: "etcdv2" - etcdEndpoints: ETCD_ENDPOINTS_REPLACE \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json deleted file mode 100644 index 052fe73..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/daemon.json +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -{ - "insecure-registries": ["DOCKER_REGISTRY_REPLACE"], - "cluster-store":"CLUSTER_STORE_REPLACE", - "cluster-advertise":"LOCAL_HOST_IP_REPLACE:2375", - "dns": ["YARN_DNS_HOST_REPLACE", "LOCAL_DNS_HOST_REPLACE"], - "hosts": ["tcp://LOCAL_HOST_IP_REPLACE:2375","unix:///var/run/docker.sock"] -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service deleted file mode 100644 index f444f99..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/docker/docker.service +++ /dev/null @@ -1,35 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[Unit] -Description=Docker Application Container Engine -Documentation=http://docs.docker.io - -[Service] -Environment="PATH=/usr/bin:/bin:/sbin:/usr/bin:/usr/sbin" -ExecStart=/usr/bin/dockerd --log-level=error -ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT -ExecReload=/bin/kill -s HUP $MAINPID -Restart=on-failure -RestartSec=5 -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity -Delegate=yes -KillMode=process - -[Install] -WantedBy=multi-user.target \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service deleted file mode 100644 index 5a44ce2..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/etcd/etcd.service +++ /dev/null @@ -1,40 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -[Unit] -Description=etcd -Documentation=https://github.com/coreos/etcd - -[Service] -Type=notify -Restart=always -RestartSec=5s -LimitNOFILE=40000 -TimeoutStartSec=0 - -ExecStart=/usr/bin/etcd \ - --name=ETCD_NODE_NAME_REPLACE \ - --data-dir=/var/lib/etcd \ - --listen-client-urls=http://LOCAL_HOST_REPLACE:2379,http://127.0.0.1:2379 \ - --listen-peer-urls=http://LOCAL_HOST_REPLACE:2380 \ - --advertise-client-urls=http://LOCAL_HOST_REPLACE:2379 \ - --initial-advertise-peer-urls=http://LOCAL_HOST_REPLACE:2380 \ - --initial-cluster=INITIAL_CLUSTER_REPLACE \ - --initial-cluster-token=etcd-token \ - --initial-cluster-state=new - -[Install] -WantedBy=multi-user.target \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg deleted file mode 100644 index 9d9f0ca..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/hadoop/container-executor.cfg +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -yarn.nodemanager.local-dirs=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE -yarn.nodemanager.linux-container-executor.group=yarn -yarn.nodemanager.log-dirs=YARN_NODEMANAGER_LOG_DIRS_REPLACE -banned.users=root -allowed.system.users=yarn -min.user.id=500 - -[docker] -module.enabled=true -docker.binary=/usr/bin/docker -docker.allowed.capabilities=SYS_CHROOT,MKNOD,SETFCAP,SETPCAP,FSETID,CHOWN,AUDIT_WRITE,SETGID,NET_RAW,FOWNER,SETUID,DAC_OVERRIDE,KILL,NET_BIND_SERVICE,DAC_READ_SEARCH,SYS_PTRACE,SYS_ADMIN -docker.allowed.networks=bridge,host,none,CALICO_NETWORK_NAME_REPLACE -docker.allowed.ro-mounts=/etc/group,/etc/passwd,/etc/krb5.conf,YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,regex:^nvidia_driver_.*$ -docker.allowed.rw-mounts=YARN_NODEMANAGER_LOCAL_DIRS_REPLACE,YARN_NODEMANAGER_LOG_DIRS_REPLACE -docker.privileged-containers.enabled=false -docker.trusted.registries=local,centos,hortonworks,DOCKER_REGISTRY_REPLACE -docker.allowed.volume-drivers=nvidia-docker -docker.allowed.devices=regex:^/dev/nvidia.*$ - -[gpu] -module.enabled=true - -[cgroups] -root=/sys/fs/cgroup -yarn-hierarchy=YARN_HIERARCHY_REPLACE \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh deleted file mode 100644 index ffbc38f..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/package/submarine/submarine.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Solution: Grant user yarn the access to /sys/fs/cgroup/cpu,cpuacct, -# which is the subfolder of cgroup mount destination. -chown :yarn -R /sys/fs/cgroup/cpu,cpuacct -chmod g+rwx -R /sys/fs/cgroup/cpu,cpuacct - -# If GPUs are usedï¼the access to cgroup devices folder is neede as well -chown :yarn -R /sys/fs/cgroup/devices -chmod g+rwx -R /sys/fs/cgroup/devices http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh deleted file mode 100644 index bd40fcf..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/calico.sh +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description download calico bin -## @audience public -## @stability stable -function download_calico_bin() -{ - # submarin http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_CALICOCTL_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calicoctl - MY_CALICO_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico - MY_CALICO_IPAM_DOWNLOAD_URL=${DOWNLOAD_HTTP}/downloads/calico/calico-ipam - else - MY_CALICOCTL_DOWNLOAD_URL=${CALICOCTL_DOWNLOAD_URL} - MY_CALICO_DOWNLOAD_URL=${CALICO_DOWNLOAD_URL} - MY_CALICO_IPAM_DOWNLOAD_URL=${CALICO_IPAM_DOWNLOAD_URL} - fi - - mkdir -p ${DOWNLOAD_DIR}/calico - - if [[ -f ${DOWNLOAD_DIR}/calico/calico ]]; then - echo "${DOWNLOAD_DIR}/calico/calico is exist." - else - echo "download ${MY_CALICO_DOWNLOAD_URL} ..." - wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_DOWNLOAD_URL} - fi - - if [[ -f ${DOWNLOAD_DIR}/calico/calicoctl ]]; then - echo "${DOWNLOAD_DIR}/calico is exist." - else - echo "download ${MY_CALICOCTL_DOWNLOAD_URL} ..." - wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICOCTL_DOWNLOAD_URL} - fi - - if [[ -f ${DOWNLOAD_DIR}/calico/calico-ipam ]]; then - echo "${DOWNLOAD_DIR}/calico/calico-ipam is exist." - else - echo "download ${MY_CALICO_IPAM_DOWNLOAD_URL} ..." - wget -P ${DOWNLOAD_DIR}/calico ${MY_CALICO_IPAM_DOWNLOAD_URL} - fi -} - -## @description install calico bin -## @audience public -## @stability stable -function install_calico_bin() -{ - download_calico_bin - - cp -f ${DOWNLOAD_DIR}/calico/calico /usr/bin/calico - cp -f ${DOWNLOAD_DIR}/calico/calicoctl /usr/bin/calicoctl - cp -f ${DOWNLOAD_DIR}/calico/calico-ipam /usr/bin/calico-ipam - - chmod +x /usr/bin/calico - chmod +x /usr/bin/calicoctl - chmod +x /usr/bin/calico-ipam -} - -## @description install calico config -## @audience public -## @stability stable -function install_calico_config() -{ - mkdir -p /etc/calico - - cp -rf ${PACKAGE_DIR}/calico ${INSTALL_TEMP_DIR}/ - - # 1. replace etcdEndpoints - # etcdEndpoints: https://10.196.69.173:2379,https://10.196.69.174:2379,https://10.196.69.175:2379 - etcdEndpoints='' - index=0 - etcdHostsSize=${#ETCD_HOSTS[@]} - for item in ${ETCD_HOSTS[@]} - do - index=$(($index+1)) - etcdEndpoints="${etcdEndpoints}http:\/\/${item}:2379" - if [[ ${index} -lt ${etcdHostsSize} ]]; then - etcdEndpoints=${etcdEndpoints}"," - fi - done - # echo "etcdEndpoints=${etcdEndpoints}" - sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calicoctl.cfg >>$LOG - - if [[ ! -d /etc/calico ]]; then - mkdir /etc/calico - else - rm -rf /etc/calico/* - fi - - cp -f $INSTALL_TEMP_DIR/calico/calicoctl.cfg /etc/calico/calicoctl.cfg - - sed -i "s/ETCD_ENDPOINTS_REPLACE/${etcdEndpoints}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG - sed -i "s/CALICO_IPV4POOL_CIDR_REPLACE/${CALICO_IPV4POOL_CIDR}/g" $INSTALL_TEMP_DIR/calico/calico-node.service >>$LOG - cp $INSTALL_TEMP_DIR/calico/calico-node.service /etc/systemd/system/ >>$LOG - - systemctl daemon-reload - systemctl enable calico-node.service -} - -## @description modify kernel network config -## @audience public -## @stability stable -function kernel_network_config() -{ - if [ `grep -c "net.ipv4.conf.all.rp_filter=1" /etc/sysctl.conf` -eq '0' ]; then - echo "net.ipv4.conf.all.rp_filter=1" >>/etc/sysctl.conf - fi - - if [ `grep -c "net.ipv4.ip_forward=1" /etc/sysctl.conf` -eq '0' ]; then - echo "net.ipv4.ip_forward=1" >>/etc/sysctl.conf - fi - - sysctl -p -} - -## @description check if the calico-network exist -## @audience public -## @stability stable -function calico_network_exist() -{ - local dockerNetwokInfo=`docker network ls --filter NAME=${CALICO_NETWORK_NAME}` - echo ${dockerNetwokInfo} | grep ${CALICO_NETWORK_NAME} -} - -## @description verification calico -## @audience public -## @stability stable -function verification_calico() -{ - echo " ===== Check if the network between 2 containers can be connected =====" - local claicoNetworkExist=`calico_network_exist` - if [[ "$claicoNetworkExist" = "" ]]; then - echo "Create a calico network" - docker network create --driver calico --ipam-driver calico-ipam ${CALICO_NETWORK_NAME} - else - echo "calico network ${CALICO_NETWORK_NAME} is exist." - fi - - local verifyA="verify-calico-network-A" - local verifyAInfo=`containers_exist ${verifyA}` - if [[ -n "$verifyAInfo" ]]; then - echo "Delete existing container ${verifyA}." - docker stop ${verifyA} - docker rm ${verifyA} - fi - echo "Create containers verify-calico-network-A" - docker run --net ${CALICO_NETWORK_NAME} --name ${verifyA} -tid busybox - - local verifyB="verify-calico-network-B" - local verifyBInfo=`containers_exist ${verifyB}` - if [[ -n "$verifyBInfo" ]]; then - echo "Delete existing container ${verifyB}." - docker stop ${verifyB} - docker rm ${verifyB} - fi - echo "Create containers verify-calico-network-B" - docker run --net ${CALICO_NETWORK_NAME} --name ${verifyB} -tid busybox - - echo -e "\033[33m${verifyA} ping ${verifyB}\033[0m" - docker exec ${verifyA} ping ${verifyB} -c 5 -} - -## @description install calico -## @audience public -## @stability stable -function install_calico() -{ - kernel_network_config - install_calico_bin - install_calico_config - start_calico - verification_calico -} - -## @description uninstall calico -## @audience public -## @stability stable -function uninstall_calico() -{ - echo "stop calico-node.service" - systemctl stop calico-node.service - - echo "rm /usr/bin/calico ..." - rm /usr/bin/calicoctl - rm /usr/bin/calico - rm /usr/bin/calico-ipam - - rm -rf /etc/calico/ - rm /etc/systemd/system/calico-node.service - systemctl daemon-reload -} - -## @description start calico -## @audience public -## @stability stable -function start_calico() -{ - systemctl restart calico-node.service - systemctl status calico-node.service -} - -## @description stop calico -## @audience public -## @stability stable -function stop_calico() -{ - systemctl stop calico-node.service - systemctl status calico-node.service -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh deleted file mode 100644 index 26f19cc..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/docker.sh +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description download docker rmp -## @audience public -## @stability stable -function download_docker_rpm() -{ - # download http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_DOCKER_ENGINE_SELINUX_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_SELINUX_RPM}" - MY_DOCKER_ENGINE_RPM="${DOWNLOAD_HTTP}/downloads/docker/${DOCKER_ENGINE_RPM}" - else - MY_DOCKER_ENGINE_SELINUX_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_SELINUX_RPM} - MY_DOCKER_ENGINE_RPM=${DOCKER_REPO}/${DOCKER_ENGINE_RPM} - fi - - # download docker rpm - if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} ]]; then - echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} is exist." - else - echo "download ${MY_DOCKER_ENGINE_SELINUX_RPM} ..." - wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_SELINUX_RPM} - fi - - if [[ -f ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} ]]; then - echo "${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} is exist." - else - echo "download ${MY_DOCKER_ENGINE_RPM} ..." - wget -P ${DOWNLOAD_DIR}/docker/ ${MY_DOCKER_ENGINE_RPM} - fi -} - -## @description install docker bin -## @audience public -## @stability stable -function install_docker_bin() -{ - download_docker_rpm - - yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} - yum -y localinstall ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} -} - -## @description uninstall docker bin -## @audience public -## @stability stable -function uninstall_docker_bin() -{ - download_docker_rpm - - yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_SELINUX_RPM} - yum -y remove ${DOWNLOAD_DIR}/docker/${DOCKER_ENGINE_RPM} -} - -## @description install docker config -## @audience public -## @stability stable -function install_docker_config() -{ - rm -rf ${INSTALL_TEMP_DIR}/docker - cp -rf ${PACKAGE_DIR}/docker ${INSTALL_TEMP_DIR}/ - - # replace cluster-store - # "cluster-store":"etcd://10.196.69.173:2379,10.196.69.174:2379,10.196.69.175:2379" - # char '/' need to escape '\/' - clusterStore="etcd:\/\/" - index=1 - etcdHostsSize=${#ETCD_HOSTS[@]} - for item in ${ETCD_HOSTS[@]} - do - clusterStore="${clusterStore}${item}:2379" - if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then - clusterStore=${clusterStore}"," - fi - index=$(($index+1)) - done - echo "clusterStore=${clusterStore}" - sed -i "s/CLUSTER_STORE_REPLACE/${clusterStore}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG - - sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG - sed -i "s/LOCAL_HOST_IP_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG - sed -i "s/YARN_DNS_HOST_REPLACE/${YARN_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG - sed -i "s/LOCAL_DNS_HOST_REPLACE/${LOCAL_DNS_HOST}/g" $INSTALL_TEMP_DIR/docker/daemon.json >>$LOG - - # Delete the ASF license comment in the daemon.json file, otherwise it will cause a json format error. - sed -i '1,16d' $INSTALL_TEMP_DIR/docker/daemon.json - - if [ ! -d "/etc/docker" ]; then - mkdir /etc/docker - fi - - cp $INSTALL_TEMP_DIR/docker/daemon.json /etc/docker/ - cp $INSTALL_TEMP_DIR/docker/docker.service /etc/systemd/system/ >>$LOG -} - -## @description install docker -## @audience public -## @stability stable -function install_docker() -{ - install_docker_bin - install_docker_config - - systemctl daemon-reload - systemctl enable docker.service -} - -## @description unstall docker -## @audience public -## @stability stable -function uninstall_docker() -{ - echo "stop docker service" - systemctl stop docker - - echo "remove docker" - uninstall_docker_bin - - rm /etc/docker/daemon.json >>$LOG - rm /etc/systemd/system/docker.service >>$LOG - - systemctl daemon-reload -} - -## @description start docker -## @audience public -## @stability stable -function start_docker() -{ - systemctl restart docker - systemctl status docker - docker info -} - -## @description stop docker -## @audience public -## @stability stable -function stop_docker() -{ - systemctl stop docker - systemctl status docker -} - -## @description check if the containers exist -## @audience public -## @stability stable -function containers_exist() -{ - local dockerContainersInfo=`docker ps -a --filter NAME=$1` - echo ${dockerContainersInfo} | grep $1 -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh deleted file mode 100644 index 62af827..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/download-server.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description start download server -## @audience public -## @stability stable -function start_download_server() -{ - if [[ "$DOWNLOAD_SERVER_IP" != "$LOCAL_HOST_IP" ]]; then - echo -e "\033[31mERROR: Only $DOWNLOAD_SERVER_IP can start the download service.\033[0m" - return 1 - fi - - echo -e "You can put the install package file in the \033[34m${DOWNLOAD_DIR}\033[0m folder first, Or automatic download." - echo -n "Do you want to start download http server?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - download_etcd_bin - download_calico_bin - download_docker_rpm - download_nvidia_driver - download_nvidia_docker_bin - download_yarn_container_executor - - python -m SimpleHTTPServer ${DOWNLOAD_SERVER_PORT} - fi -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh deleted file mode 100644 index 17f59b2..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/environment.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description check operation System -## @audience public -## @stability stable -function check_operationSystem() -{ - echo -e "The submarine assembly support \033[32m[centos-release-7-3.1611.el7.centos.x86_64]\033[0m or higher operating system version." - - case ${OPERATING_SYSTEM} in - centos) - local operationSystemVersion=`rpm --query centos-release` - echo -e "The current operating system version is \e[31m[${operationSystemVersion}]\e[0m" | tee -a $LOG - ;; - *) - echo -e "\033[31mWARN: The submarine assembly Unsupported [${OPERATING_SYSTEM}] operating system\033[0m" - ;; - esac -} - -## @description update operation System Kernel -## @audience public -## @stability stable -function update_operationSystemKernel() -{ - echo "If the server is unable to connect to the network, execute the following command yourself: - wget http://vault.centos.org/7.3.1611/os/x86_64/Packages/kernel-headers-3.10.0-514.el7.x86_64.rpm - rpm -ivh kernel-headers-3.10.0-514.el7.x86_64.rpm" - - echo -n "Do you want to kernel upgrades?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - echo "Now try to use the yum command for kernel upgrades ..." - yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) - - local kernelVersion=`uname -r` - echo -e "After the upgrade, the operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG - fi -} - -## @description check operation system kernel -## @audience public -## @stability stable -function check_operationSystemKernel() -{ -case ${OPERATING_SYSTEM} in -centos) - local kernelVersion=`uname -r` - - echo -e "Submarine support operating system kernel version is \033[32m 3.10.0-514.el7.x86_64 \033[0m" | tee -a $LOG - echo -e "Current operating system kernel version is \e[31m${kernelVersion}\e[0m" | tee -a $LOG - - update_operationSystemKernel - ;; -*) - echo -e "\033[31m WARN: The submarine assembly Unsupported operating system [${OPERATING_SYSTEM}] \033[0m" - ;; -esac -} - -## @description get gcc version -## @audience public -## @stability stable -function get_gcc_version() -{ - local gccVersion=`gcc --version` - version=${gccVersion%Copyright*} - echo $version -} - -## @description install gcc -## @audience public -## @stability stable -function install_gcc() -{ - echo -n "Do you want to install gcc?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]]; then - echo "Execute the yum install gcc make g++ command" - yum install gcc make g++ - - local gccVersion=`gcc --version` - echo -e "After the install, the gcc version is \e[31m${gccVersion}\e[0m" | tee -a $LOG - fi -} - -## @description check gcc Version -## @audience public -## @stability stable -function check_gccVersion() -{ - local gccVersionInfo=`gcc --version` - local gccVersion=${gccVersionInfo%Copyright*} - - if [[ "$gccVersion" = "" ]]; then - echo "The gcc was not installed on the system. Automated installation ..." - install_gcc - else - echo -e "Submarine gcc version need \033[34mgcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-11)\033[0m or higher." - echo -e "Current gcc version was \033[34m${gccVersion}\033[0m" - fi -} - -## @description check GPU -## @audience public -## @stability stable -function check_GPU() -{ - gpuInfo=`lspci | grep -i nvidia` - - if [[ "$gpuInfo" = "" ]]; then - echo -e "\033[31mERROR: The system did not detect the GPU graphics card.\033[0m" - else - echo -e "\033[32mINFO: The system detect the GPU graphics card.\033[0m" - fi -} - -## @description check user group -## @audience public -## @stability stable -function check_userGroup() -{ - echo -e "check hadoop user group ..." - - echo -e "Hadoop runs the required user [hdfs, mapred, yarn] and groups [hdfs, mapred, yarn, hadoop] installed by ambari." - echo -e "If you are not using ambari for hadoop installation, -then you can add the user and group by root by executing the following command: -\033[34madduser hdfs -adduser mapred -adduser yarn -addgroup hadoop -usermod -aG hdfs,hadoop hdfs -usermod -aG mapred,hadoop mapred -usermod -aG yarn,hadoop yarn -usermod -aG hdfs,hadoop hadoop -groupadd docker -usermod -aG docker yarn -usermod -aG docker hadoop\033[0m\n" - - echo -e "check docker user group ..." - # check user group - DOCKER_USER_GROUP='docker' - egrep "^${DOCKER_USER_GROUP}" /etc/group >& /dev/null - if [[ $? -ne 0 ]]; then - echo -e "user group ${DOCKER_USER_GROUP} does not exist, Please execute the following command:" - echo -e "\033[34mgroupadd $DOCKER_USER_GROUP\033[0m" - fi - - # check user - USER_GROUP=(yarn hadoop) - for user in ${USER_GROUP[@]} - do - egrep "^${user}" /etc/passwd >& /dev/null - if [[ $? -ne 0 ]]; then - echo -e "User ${user} does not exist, Please execute the following command:" - echo -e "\033[34madduser ${user}\033[0m" - echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m" - fi - - echo -e "Please execute the following command:" - echo -e "\033[34musermod -aG ${DOCKER_USER_GROUP} ${user}\033[0m" - done -} - -## @description Some preparatory work for nvidia driver installation -## @audience public -## @stability stable -function prepare_nvidia_environment() -{ - echo "prepare nvidia environment ..." - - yum -y update - yum install kernel-devel-$(uname -r) kernel-headers-$(uname -r) - - yum -y install epel-release - yum -y install dkms - - echo -e "\033[34m ===== Please manually execute the following command ===== -# 1. Disable nouveau -# Add the content 'rd.driver.blacklist=nouveau nouveau.modeset=0' -# to the 'GRUB_CMDLINE_LINUX' configuration item in the /etc/default/grub file. -root:> vi /etc/default/grub -vi:> GRUB_CMDLINE_LINUX=\"rd.driver.blacklist=nouveau nouveau.modeset=0 ...\" - -# 2. Generate configuration -root:> grub2-mkconfig -o /boot/grub2/grub.cfg - -# 3. Open (new) /etc/modprobe.d/blacklist.conf, add content 'blacklist nouveau' -root:> vi /etc/modprobe.d/blacklist.conf -vi:> blacklist nouveau - -# 4. Update configuration and reboot -root:> mv /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r)-nouveau.img -root:> dracut /boot/initramfs-$(uname -r).img $(uname -r) -root:> reboot -\033[0m" -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh deleted file mode 100644 index 48df80e..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/etcd.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description download etcd bin -## @audience public -## @stability stable -function download_etcd_bin() -{ - # my download http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_ETCD_DOWNLOAD_URL="${DOWNLOAD_HTTP}/downloads/etcd/${ETCD_TAR_GZ}" - else - MY_ETCD_DOWNLOAD_URL=${ETCD_DOWNLOAD_URL} - fi - - if [[ -f "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ}" ]]; then - echo "${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} is exist." - else - echo "download ${MY_ETCD_DOWNLOAD_URL} ..." - wget -P ${DOWNLOAD_DIR}/etcd ${MY_ETCD_DOWNLOAD_URL} - fi -} - -## @description install etcd bin -## @audience public -## @stability stable -function install_etcd_bin() -{ - download_etcd_bin - - # install etcd bin - mkdir -p ${INSTALL_TEMP_DIR} - rm -rf ${INSTALL_TEMP_DIR}/etcd-*-linux-amd6 - tar zxvf ${DOWNLOAD_DIR}/etcd/${ETCD_TAR_GZ} -C ${INSTALL_TEMP_DIR} - - cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcd /usr/bin - cp -f ${INSTALL_TEMP_DIR}/etcd-*-linux-amd64/etcdctl /usr/bin - - mkdir -p /var/lib/etcd - chmod -R a+rw /var/lib/etcd -} - -## @description install etcd config -## @audience public -## @stability stable -function install_etcd_config() -{ - # config etcd.service - rm -rf ${INSTALL_TEMP_DIR}/etcd - cp -rf ${PACKAGE_DIR}/etcd ${INSTALL_TEMP_DIR}/ - - # 1. Replace name with ETCD_NODE_NAME_REPLACE based on the location of the local IP in $ETCD_HOSTS - indexEtcdList=$(indexByEtcdHosts ${LOCAL_HOST_IP}) - # echo ${indexEtcdList} - etcdNodeName="etcdnode${indexEtcdList}" - # echo ${etcdNodeName} - sed -i "s/ETCD_NODE_NAME_REPLACE/${etcdNodeName}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG - - # 2. Replace local IP address - sed -i "s/LOCAL_HOST_REPLACE/${LOCAL_HOST_IP}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG - - # 3. Replace the initial-cluster parameter - # --initial-cluster=etcdnode1=http://10.196.69.173:2380,etcdnode2=http://10.196.69.174:2380,etcdnode3=http://10.196.69.175:2380 \ - initialCluster='' - index=0 - etcdHostsSize=${#ETCD_HOSTS[@]} - for item in ${ETCD_HOSTS[@]} - do - # char '/' need to escape '\/' - initialCluster="${initialCluster}etcdnode${index}=http:\/\/${item}:2380" - if [[ ${index} -lt ${etcdHostsSize}-1 ]]; then - initialCluster=${initialCluster}"," - fi - index=$(($index+1)) - done - #echo "initialCluster=${initialCluster}" - sed -i "s/INITIAL_CLUSTER_REPLACE/${initialCluster}/g" $INSTALL_TEMP_DIR/etcd/etcd.service >>$LOG - - cp $INSTALL_TEMP_DIR/etcd/etcd.service /etc/systemd/system/ >>$LOG -} - -## @description install etcd -## @audience public -## @stability stable -function install_etcd() -{ - index=$(indexByEtcdHosts ${LOCAL_HOST_IP}) - if [ -z "$index" ]; then - echo -e "STOP: This host\033[31m[${LOCAL_HOST_IP}]\033[0m is not in the ETCD server list\033[31m[${ETCD_HOSTS[@]}]\033[0m" - return 1 - fi - - install_etcd_bin - - install_etcd_config - - systemctl daemon-reload - systemctl enable etcd.service -} - -## @description uninstall etcd -## @audience public -## @stability stable -function uninstall_etcd() -{ - echo "stop etcd.service" - systemctl stop etcd.service - - echo "rm etcd ..." - rm /usr/bin/etcd - rm /usr/bin/etcdctl - rm -rf /var/lib/etcd - rm /etc/systemd/system/etcd.service - - systemctl daemon-reload -} - -## @description start etcd -## @audience public -## @stability stable -function start_etcd() -{ - systemctl restart etcd.service - - echo " ===== Check the status of the etcd service =====" - echo " exec etcdctl cluster-health" - etcdctl cluster-health - echo " exec etcdctl cluster-health" - etcdctl member list -} - -## @description stop etcd -## @audience public -## @stability stable -function stop_etcd() -{ - systemctl stop etcd.service -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh deleted file mode 100644 index 9df5d7e..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/hadoop.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description install yarn -## @audience public -## @stability stable -function install_yarn() -{ - install_yarn_container_executor - install_yarn_config -} - -## @description uninstall yarn -## @audience public -## @stability stable -function uninstall_yarn() -{ - rm -rf /etc/yarn/sbin/Linux-amd64-64/* - rm -rf /etc/yarn/sbin/etc/hadoop/* -} - -## @description download yarn container executor -## @audience public -## @stability stable -function download_yarn_container_executor() -{ - # my download http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_YARN_CONTAINER_EXECUTOR_PATH="${DOWNLOAD_HTTP}/downloads/hadoop/container-executor" - else - MY_YARN_CONTAINER_EXECUTOR_PATH=${YARN_CONTAINER_EXECUTOR_PATH} - fi - - if [ ! -d "${DOWNLOAD_DIR}/hadoop" ]; then - mkdir -p ${DOWNLOAD_DIR}/hadoop - fi - - if [[ -f "${DOWNLOAD_DIR}/hadoop/container-executor" ]]; then - echo "${DOWNLOAD_DIR}/hadoop/container-executor is exist." - else - if [[ -n "$DOWNLOAD_HTTP" ]]; then - echo "download ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..." - wget -P ${DOWNLOAD_DIR}/hadoop ${MY_YARN_CONTAINER_EXECUTOR_PATH} - else - echo "copy ${MY_YARN_CONTAINER_EXECUTOR_PATH} ..." - cp ${MY_YARN_CONTAINER_EXECUTOR_PATH} ${DOWNLOAD_DIR}/hadoop/ - fi - fi -} - -## @description install yarn container executor -## @audience public -## @stability stable -function install_yarn_container_executor() -{ - echo "install yarn container executor file ..." - - download_yarn_container_executor - - if [ ! -d "/etc/yarn/sbin/Linux-amd64-64" ]; then - mkdir -p /etc/yarn/sbin/Linux-amd64-64 - fi - if [ -f "/etc/yarn/sbin/Linux-amd64-64/container-executor" ]; then - rm /etc/yarn/sbin/Linux-amd64-64/container-executor - fi - - cp -f ${DOWNLOAD_DIR}/hadoop/container-executor /etc/yarn/sbin/Linux-amd64-64 - - sudo chmod 6755 /etc/yarn/sbin/Linux-amd64-64 - sudo chown :yarn /etc/yarn/sbin/Linux-amd64-64/container-executor - sudo chmod 6050 /etc/yarn/sbin/Linux-amd64-64/container-executor -} - -## @description install yarn config -## @audience public -## @stability stable -function install_yarn_config() -{ - echo "install yarn config file ..." - - cp -R ${PACKAGE_DIR}/hadoop ${INSTALL_TEMP_DIR}/ - - find="/" - replace="\/" - escape_yarn_nodemanager_local_dirs=${YARN_NODEMANAGER_LOCAL_DIRS//$find/$replace} - escape_yarn_nodemanager_log_dirs=${YARN_NODEMANAGER_LOG_DIRS//$find/$replace} - escape_yarn_hierarchy=${YARN_HIERARCHY//$find/$replace} - - sed -i "s/YARN_NODEMANAGER_LOCAL_DIRS_REPLACE/${escape_yarn_nodemanager_local_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG - sed -i "s/YARN_NODEMANAGER_LOG_DIRS_REPLACE/${escape_yarn_nodemanager_log_dirs}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG - sed -i "s/DOCKER_REGISTRY_REPLACE/${DOCKER_REGISTRY}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG - sed -i "s/CALICO_NETWORK_NAME_REPLACE/${CALICO_NETWORK_NAME}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG - sed -i "s/YARN_HIERARCHY_REPLACE/${escape_yarn_hierarchy}/g" $INSTALL_TEMP_DIR/hadoop/container-executor.cfg >>$LOG - - # Delete the ASF license comment in the container-executor.cfg file, otherwise it will cause a cfg format error. - sed -i '1,16d' $INSTALL_TEMP_DIR/hadoop/container-executor.cfg - - if [ ! -d "/etc/yarn/sbin/etc/hadoop" ]; then - mkdir -p /etc/yarn/sbin/etc/hadoop - fi - - cp -f $INSTALL_TEMP_DIR/hadoop/container-executor.cfg /etc/yarn/sbin/etc/hadoop/ -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh deleted file mode 100644 index 68f91c1..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/menu.sh +++ /dev/null @@ -1,444 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description main menu -## @audience public -## @stability stable -main_menu() -{ -cat<<MENULIST -==================================================================================== - SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} - -HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} -==================================================================================== -[Main menu] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.prepare system environment [..]\e[0m" -echo -e " \e[32m2.install component [..]\e[0m" -echo -e " \e[32m3.uninstall component [..]\e[0m" -echo -e " \e[32m4.start component [..]\e[0m" -echo -e " \e[32m5.stop component [..]\e[0m" -echo -e " \e[32m6.start download server [..]\e[0m" -echo -e "" -echo -e " \e[32mq.quit\e[0m" -cat<<MENULIST -==================================================================================== -MENULIST - -echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m6\e[0m,\e[32mq\e[0m(quit)]:" -} - -## @description check menu -## @audience public -## @stability stable -check_menu() -{ -cat<<MENULIST -==================================================================================== - SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} - -HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} -==================================================================================== -[Main menu] > [prepare system environment] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.prepare operation system\e[0m" -echo -e " \e[32m2.prepare operation system kernel\e[0m" -echo -e " \e[32m3.prepare GCC version\e[0m" -echo -e " \e[32m4.check GPU\e[0m" -echo -e " \e[32m5.prepare user&group\e[0m" -echo -e " \e[32m6.prepare nvidia environment\e[0m" -echo -e "" -echo -e " \e[32mb.back main menu\e[0m" -cat<<MENULIST -==================================================================================== -MENULIST - -echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m5\e[0m,\e[32mb\e[0m(back)]:" -} - -## @description install menu -## @audience public -## @stability stable -install_menu() -{ -cat<<MENULIST -==================================================================================== - SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} - -HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} -==================================================================================== -[Main menu] > [install component] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.instll etcd\e[0m" -echo -e " \e[32m2.instll docker\e[0m" -echo -e " \e[32m3.instll calico network\e[0m" -echo -e " \e[32m4.instll nvidia driver\e[0m" -echo -e " \e[32m5.instll nvidia docker\e[0m" -echo -e " \e[32m6.instll yarn container-executor\e[0m" -echo -e " \e[32m7.instll submarine autorun script\e[0m" -echo -e "" -echo -e " \e[32mb.back main menu\e[0m" -cat<<MENULIST -==================================================================================== -MENULIST - -echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m7\e[0m,\e[32mb\e[0m(back)]:" -} - -## @description unstall menu -## @audience public -## @stability stable -uninstall_menu() -{ -cat<<MENULIST -==================================================================================== - SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} - -HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} -==================================================================================== -[Main menu] > [uninstll component] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.uninstll etcd\e[0m" -echo -e " \e[32m2.uninstll docker\e[0m" -echo -e " \e[32m3.uninstll calico network\e[0m" -echo -e " \e[32m4.uninstll nvidia driver\e[0m" -echo -e " \e[32m5.uninstll nvidia docker\e[0m" -echo -e " \e[32m6.uninstll yarn container-executor\e[0m" -echo -e " \e[32m7.uninstll submarine autorun script\e[0m" -echo -e "" -echo -e " \e[32mb.back main menu\e[0m" -cat<<MENULIST -==================================================================================== -MENULIST - -echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m7\e[0m,\e[32mb\e[0m(back)]:" -} - -## @description start menu -## @audience public -## @stability stable -start_menu() -{ -cat<<MENULIST -==================================================================================== - SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} - -HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} -==================================================================================== -[Main menu] > [stop component] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.start etcd\e[0m" -echo -e " \e[32m2.start docker\e[0m" -echo -e " \e[32m3.start calico network\e[0m" -echo -e "" -echo -e " \e[32mb.back main menu\e[0m" -cat<<MENULIST -==================================================================================== -MENULIST - -echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m3\e[0m,\e[32mb\e[0m(back)]:" -} - -## @description stop menu -## @audience public -## @stability stable -stop_menu() -{ -cat<<MENULIST -==================================================================================== - SUBMARINE INSTALLER ${SUBMARINE_INSTALLER_VERSION} - -HOST:${LOCAL_HOST_IP} DOWNLOAD_SERVER:http://${DOWNLOAD_SERVER_IP}:${DOWNLOAD_SERVER_PORT} -==================================================================================== -[Main menu] > [stop component] ------------------------------------------------------------------------------------- -MENULIST -echo -e " \e[32m1.stop etcd\e[0m" -echo -e " \e[32m2.stop docker\e[0m" -echo -e " \e[32m3.stop calico network\e[0m" -echo -e "" -echo -e " \e[32mb.back main menu\e[0m" -cat<<MENULIST -==================================================================================== -MENULIST - -echo -ne "Please input your choice [\e[32m1\e[0m-\e[32m3\e[0m,\e[32mb\e[0m(back)]:" -} - -## @description menu operation -## @audience public -## @stability stable -menu_index="0" -menu() -{ - clear - # echo "menu_index-menu_choice=$menu_index-$menu_choice" - case $menu_index in - "0") - main_menu - ;; - "1") - check_menu - ;; - "2") - install_menu - ;; - "3") - uninstall_menu - ;; - "4") - start_menu - ;; - "5") - stop_menu - ;; - "6") - start_download_server - ;; - "q") - exit 1 - ;; - *) - echo "error input!" - menu_index="0" - menu_choice="0" - main_menu - ;; - esac - - read menu_choice -} - -## @description menu process -## @audience public -## @stability stable -menu_process() -{ - process=0 - unset myselect - # echo "debug=$menu_index-$menu_choice" - case "$menu_index-$menu_choice" in - "1-b"|"2-b"|"3-b"|"4-b"|"5-b"|"6-b") - menu_index="0" - menu_choice="0" - ;; -# check system environment - "1-1") - myselect="y" - check_operationSystem - ;; - "1-2") - myselect="y" - check_operationSystemKernel - ;; - "1-3") - myselect="y" - check_gccVersion - ;; - "1-4") - myselect="y" - check_GPU - ;; - "1-5") - myselect="y" - check_userGroup - ;; - "1-6") - myselect="y" - prepare_nvidia_environment - ;; -# install component - "2-1") - echo -n "Do you want to install etcd?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - install_etcd - fi - ;; - "2-2") - echo -n "Do you want to install docker?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - install_docker - fi - ;; - "2-3") - echo -n "Do you want to install calico network?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - install_calico - fi - ;; - "2-4") - echo -n "Do you want to install nvidia driver?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - install_nvidia - fi - ;; - "2-5") - echo -n "Do you want to install nvidia docker?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - install_nvidia_docker - fi - ;; - "2-6") - echo -n "Do you want to install yarn container-executor?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - install_yarn - fi - ;; - "2-7") - echo -n "Do you want to install submarine auto start script?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - install_submarine - fi - ;; -# uninstall component - "3-1") - echo -n "Do you want to uninstall etcd?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - uninstall_etcd - fi - ;; - "3-2") - echo -n "Do you want to uninstall docker?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - uninstall_docker - fi - ;; - "3-3") - echo -n "Do you want to uninstall calico network?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - uninstall_calico - fi - ;; - "3-4") - echo -n "Do you want to uninstall nvidia driver?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - uninstall_nvidia - fi - ;; - "3-5") - echo -n "Do you want to uninstall nvidia docker?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - uninstall_nvidia_docker - fi - ;; - "3-6") - echo -n "Do you want to uninstall yarn container-executor?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - uninstall_yarn - fi - ;; - "3-7") - echo -n "Do you want to uninstall submarine autostart script?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - uninstall_submarine - fi - ;; -# startup component - "4-1") - echo -n "Do you want to startup etcd?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - start_etcd - fi - ;; - "4-2") - echo -n "Do you want to startup docker?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - start_docker - fi - ;; - "4-3") - echo -n "Do you want to startup calico network?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - start_calico - fi - ;; -# stop component - "5-1") - echo -n "Do you want to stop etcd?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - stop_etcd - fi - ;; - "5-2") - echo -n "Do you want to stop docker?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - stop_docker - fi - ;; - "5-3") - echo -n "Do you want to stop calico network?[y|n]" - read myselect - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - stop_calico - fi - ;; - esac - - if [[ "$myselect" = "y" || "$myselect" = "Y" ]] - then - process=1 - fi - -# echo "process=$process" - return $process -} - http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh deleted file mode 100644 index 3441f28..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia-docker.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description download nvidia docker bin -## @audience public -## @stability stable -function download_nvidia_docker_bin() -{ - # download http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_NVIDIA_DOCKER_RPM_URL="${DOWNLOAD_HTTP}/downloads/nvidia-docker/${NVIDIA_DOCKER_RPM}" - else - MY_NVIDIA_DOCKER_RPM_URL=${NVIDIA_DOCKER_RPM_URL} - fi - - if [[ -f "${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM}" ]]; then - echo "${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM} is exist." - else - echo "download ${MY_NVIDIA_DOCKER_RPM_URL} ..." - wget -P ${DOWNLOAD_DIR}/nvidia-docker/ ${MY_NVIDIA_DOCKER_RPM_URL} - fi -} - -## @description install nvidia docker -## @audience public -## @stability stable -function install_nvidia_docker() -{ - download_nvidia_docker_bin - - sudo rpm -i ${DOWNLOAD_DIR}/nvidia-docker/${NVIDIA_DOCKER_RPM} - - echo -e "\033[32m===== Start nvidia-docker =====\033[0m" - sudo systemctl start nvidia-docker - - echo -e "\033[32m===== Check nvidia-docker status =====\033[0m" - systemctl status nvidia-docker - - echo -e "\033[32m===== Check nvidia-docker log =====\033[0m" - journalctl -u nvidia-docker - - echo -e "\033[32m===== Test nvidia-docker-plugin =====\033[0m" - curl http://localhost:3476/v1.0/docker/cli - - # create nvidia driver library path - if [ ! -d "/var/lib/nvidia-docker/volumes/nvidia_driver" ]; then - echo "WARN: /var/lib/nvidia-docker/volumes/nvidia_driver folder path is not exist!" - mkdir -p /var/lib/nvidia-docker/volumes/nvidia_driver - fi - - local nvidiaVersion=`get_nvidia_version` - echo -e "\033[31m nvidia detect version is ${nvidiaVersion}\033[0m" - - mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion} - mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/bin - mkdir /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64 - - cp /usr/bin/nvidia* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/bin - cp /usr/lib64/libcuda* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64 - cp /usr/lib64/libnvidia* /var/lib/nvidia-docker/volumes/nvidia_driver/${nvidiaVersion}/lib64 - - echo -e "\033[32m===== Please manually execute the following command =====\033[0m" - echo -e "\033[32mshell:> nvidia-docker run --rm ${DOCKER_REGISTRY}/nvidia/cuda:9.0-devel nvidia-smi -# If you don't see the list of graphics cards above, the NVIDIA driver installation failed. ===== -\033[0m" - - echo -e "\033[32m===== Please manually execute the following command =====\033[0m" - echo -e "\033[32m# Test with tf.test.is_gpu_available() -shell:> nvidia-docker run -it ${DOCKER_REGISTRY}/tensorflow/tensorflow:1.9.0-gpu bash -# In docker container -container:> python -python:> import tensorflow as tf -python:> tf.test.is_gpu_available() -python:> exit() -\033[0m" -} - -## @description uninstall nvidia docker -## @audience public -## @stability stable -function uninstall_nvidia_docker() -{ - echo "This method is not implemented." -} - http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh deleted file mode 100644 index 7b2b1de..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/nvidia.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -nvidia_run_file="" - -## @description get nvidia version -## @audience public -## @stability stable -function get_nvidia_version() -{ - chmod +x ${DOWNLOAD_DIR}/nvidia/nvidia-detect - local nvidia_detect_info=`${DOWNLOAD_DIR}/nvidia/nvidia-detect -v` - echo $nvidia_detect_info | sed "s/^.*This device requires the current \([0-9.]*\).*/\1/" -} - -## @description download nvidia driver -## @audience public -## @stability stable -function download_nvidia_driver() -{ - # submarin http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_NVIDIA_DETECT_URL="${DOWNLOAD_HTTP}/downloads/nvidia/nvidia-detect" - else - MY_NVIDIA_DETECT_URL=${NVIDIA_DETECT_URL} - fi - - # download nvidia_detect - if [[ -f "${DOWNLOAD_DIR}/nvidia/nvidia-detect" ]]; then - echo "${DOWNLOAD_DIR}/nvidia/nvidia-detect is exist." - else - echo "download ${MY_NVIDIA_DETECT_URL} ..." - wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DETECT_URL} - fi - - echo "execution nvidia-detect to check the graphics card ..." - local nvidiaVersion=`get_nvidia_version` - echo -e "detect nvidia version is \033[31m${nvidiaVersion}\033[0m" - - # download NVIDIA driver - if [[ "$nvidiaVersion" = "" ]]; then - echo -e "\033[31mERROR: No graphics card device detected.\033[0m" - return 1 - else - nvidia_run_file="NVIDIA-Linux-x86_64-${nvidiaVersion}.run" - - # submarin http server - if [[ -n "$DOWNLOAD_HTTP" ]]; then - MY_NVIDIA_DRIVER_RUN_URL="${DOWNLOAD_HTTP}/downloads/nvidia/${nvidia_run_file}" - else - # http://us.download.nvidia.com/XFree86/Linux-x86_64/390.87/NVIDIA-Linux-x86_64-390.87.run - MY_NVIDIA_DRIVER_RUN_URL="http://us.download.nvidia.com/XFree86/Linux-x86_64/${nvidiaVersion}/${nvidia_run_file}"; - fi - - if [[ -f ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} ]]; then - echo "NVIDIA driver files already exist in the ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} directory." - echo "===== Please make sure the ${DOWNLOAD_DIR}/nvidia/nvidia/${nvidia_run_file} file is complete and can be used normally. =====" - else - echo "Download the NVIDIA driver from the ${MY_NVIDIA_DRIVER_RUN_URL}" - wget -P ${DOWNLOAD_DIR}/nvidia/ ${MY_NVIDIA_DRIVER_RUN_URL} - fi - fi -} - -## @description install nvidia -## @audience public -## @stability stable -function install_nvidia() -{ - download_nvidia_driver - - # Confirm that the system disables nouveau - local disable_nouveau_info=`lsmod | grep nouveau` - if [[ "$disable_nouveau_info" = "" ]]; then - echo "===== Start installing the NVIDIA driver =====" - echo -e "Some options during the installation -Would you like to register the kernel module sources with DKMS? - This will allow DKMS to automatically build a new module, if you install a different kernel later. \033[33m[Yes]\033[0m -Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m -centos Install NVIDIA's 32-bit compatibility libraries \033[33m[Yes]\033[0m -Would you like to run the nvidia-xconfig utility to automatically update your X configuration file... \033[33m[No]\033[0m" - sleep 2 - sh ${DOWNLOAD_DIR}/nvidia/${nvidia_run_file} - else - echo -e "ERROR: Nouveau is not disabled" - return 1 - fi - - echo -e "\033[32m===== execute nvidia-smi. You should be able to see the list of graphics cards =====\033[0m" - sleep 1 - nvidia-smi -} - -## @description uninstall nvidia -## @audience public -## @stability stable -function uninstall_nvidia() -{ - if [ ! -f "/usr/bin/nvidia-uninstall" ]; then - echo -e "\033[31mERROR: /usr/bin/nvidia-uninstall file is not exist!\033[0m" - return 1 - fi - - echo -e "execute /usr/bin/nvidia-uninstall" - /usr/bin/nvidia-uninstall -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/228156cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh deleted file mode 100644 index b73d5e7..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-submarine/installation/scripts/submarine.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -## @description install submarine autorun script -## @audience public -## @stability stable -function install_submarine() -{ - cp ${PACKAGE_DIR}/submarine/submarine.sh /etc/rc.d/init.d/submarine.sh - chmod +x /etc/rc.d/init.d/submarine.sh - chmod +x /etc/rc.d/rc.local - - if [ `grep -c "/etc/rc.d/init.d/submarine.sh" /etc/rc.d/rc.local` -eq '0' ]; then - echo "/etc/rc.d/init.d/submarine.sh">> /etc/rc.d/rc.local - fi -} - -## @description uninstall submarine autorun script -## @audience public -## @stability stable -function uninstall_submarine() -{ - rm /etc/rc.d/init.d/submarine.sh -} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
